From 917132a19cd58b3151d6f30839eb327a26459117 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Thu, 19 Apr 2018 11:13:35 +1200 Subject: [PATCH] Function versioning API: delete When deleting a function version: - The version should not being used by any job - The version should not being used by any webhook - Function latest_version should be updated if the version in request equals to latest_version Story: #2001829 Task: #14349 Change-Id: I5511304e04bec8f404d7224a6489b65d6a765316 --- .../api/controllers/v1/function_version.py | 48 +++++++++++++++ qinling/db/api.py | 4 ++ qinling/db/sqlalchemy/api.py | 6 ++ qinling/db/sqlalchemy/models.py | 1 + qinling/storage/base.py | 4 +- qinling/storage/file_system.py | 26 +++++--- .../unit/api/controllers/v1/test_execution.py | 4 +- .../unit/api/controllers/v1/test_function.py | 57 +++++------------ .../controllers/v1/test_function_version.py | 57 ++++++++++++++++- .../controllers/v1/test_function_worker.py | 2 - .../tests/unit/api/controllers/v1/test_job.py | 12 ++-- .../unit/api/controllers/v1/test_runtime.py | 19 +++--- .../unit/api/controllers/v1/test_webhook.py | 4 +- qinling/tests/unit/base.py | 33 +++++----- .../tests/unit/engine/test_default_engine.py | 61 ++++++++----------- qinling/tests/unit/services/test_periodics.py | 11 +--- .../tests/unit/storage/test_file_system.py | 19 ++++++ 17 files changed, 227 insertions(+), 141 deletions(-) diff --git a/qinling/api/controllers/v1/function_version.py b/qinling/api/controllers/v1/function_version.py index 0ce35ff6..deaf3a98 100644 --- a/qinling/api/controllers/v1/function_version.py +++ b/qinling/api/controllers/v1/function_version.py @@ -110,6 +110,8 @@ class FunctionVersionsController(rest.RestController): def post(self, function_id, body): """Create a new version for the function. + Only allow to create version for package type function. + The supported boy params: - description: Optional. The description of the new version. """ @@ -177,3 +179,49 @@ class FunctionVersionsController(rest.RestController): pecan.response.headers['Content-Disposition'] = ( 'attachment; filename="%s_%s"' % (function_id, version) ) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(None, types.uuid, int, status_code=204) + def delete(self, function_id, version): + """Delete a specific function version. + + - The version should not being used by any job + - The version should not being used by any webhook + """ + ctx = context.get_ctx() + acl.enforce('function_version:delete', ctx) + LOG.info("Deleting version %s of function %s.", version, function_id) + + with db_api.transaction(): + version_db = db_api.get_function_version(function_id, version) + latest_version = version_db.function.latest_version + + version_jobs = db_api.get_jobs( + function_id=version_db.function_id, + function_version=version_db.version_number, + status={'nin': ['done', 'cancelled']} + ) + if len(version_jobs) > 0: + raise exc.NotAllowedException( + 'The function version is still associated with running ' + 'job(s).' + ) + + version_webhook = db_api.get_webhooks( + function_id=version_db.function_id, + function_version=version_db.version_number, + ) + if len(version_webhook) > 0: + raise exc.NotAllowedException( + 'The function versioin is still associated with webhook.' + ) + + self.storage_provider.delete(ctx.projectid, function_id, None, + version=version) + + db_api.delete_function_version(function_id, version) + + if latest_version == version: + version_db.function.latest_version = latest_version - 1 + + LOG.info("Version %s of function %s deleted.", version, function_id) diff --git a/qinling/db/api.py b/qinling/db/api.py index d9a362d7..4fc2100b 100644 --- a/qinling/db/api.py +++ b/qinling/db/api.py @@ -210,3 +210,7 @@ def increase_function_version(function_id, old_version, **kwargs): def get_function_version(function_id, version): return IMPL.get_function_version(function_id, version) + + +def delete_function_version(function_id, version): + return IMPL.delete_function_version(function_id, version) diff --git a/qinling/db/sqlalchemy/api.py b/qinling/db/sqlalchemy/api.py index c17ce8ca..f7ce3dc2 100644 --- a/qinling/db/sqlalchemy/api.py +++ b/qinling/db/sqlalchemy/api.py @@ -527,3 +527,9 @@ def get_function_version(function_id, version, session=None): ) return version_db + + +@db_base.session_aware() +def delete_function_version(function_id, version, session=None): + version_db = get_function_version(function_id, version) + session.delete(version_db) diff --git a/qinling/db/sqlalchemy/models.py b/qinling/db/sqlalchemy/models.py index cf1944f8..26217afa 100644 --- a/qinling/db/sqlalchemy/models.py +++ b/qinling/db/sqlalchemy/models.py @@ -116,6 +116,7 @@ class FunctionVersion(model_base.QinlingSecureModelBase): sa.String(36), sa.ForeignKey(Function.id, ondelete='CASCADE') ) + function = relationship('Function', back_populates="versions") description = sa.Column(sa.String(255), nullable=True) version_number = sa.Column(sa.Integer, default=0) diff --git a/qinling/storage/base.py b/qinling/storage/base.py index 299c536d..7f3fd2a9 100644 --- a/qinling/storage/base.py +++ b/qinling/storage/base.py @@ -40,7 +40,7 @@ class PackageStorage(object): raise NotImplementedError @abc.abstractmethod - def retrieve(self, project_id, function, md5sum, version=None): + def retrieve(self, project_id, function, md5sum, version=0): """Get function package data. :param project_id: Project ID. @@ -52,7 +52,7 @@ class PackageStorage(object): raise NotImplementedError @abc.abstractmethod - def delete(self, project_id, function, md5sum): + def delete(self, project_id, function, md5sum, version=0): raise NotImplementedError @abc.abstractmethod diff --git a/qinling/storage/file_system.py b/qinling/storage/file_system.py index 6969952d..3028f440 100644 --- a/qinling/storage/file_system.py +++ b/qinling/storage/file_system.py @@ -123,16 +123,28 @@ class FileSystemStorage(base.PackageStorage): return f - def delete(self, project_id, function, md5sum): + def delete(self, project_id, function, md5sum, version=0): LOG.debug( - 'Deleting package data, function: %s, md5sum: %s, project: %s', - function, md5sum, project_id + 'Deleting package data, function: %s, version: %s, md5sum: %s, ' + 'project: %s', + function, version, md5sum, project_id ) - func_zip = os.path.join( - self.base_path, - PACKAGE_PATH_TEMPLATE % (project_id, function, md5sum) - ) + if version != 0: + project_dir = os.path.join(self.base_path, project_id) + for filename in os.listdir(project_dir): + root, ext = os.path.splitext(filename) + if (root.startswith("%s_%d" % (function, version)) + and ext == '.zip'): + func_zip = os.path.join(project_dir, filename) + break + else: + return + else: + func_zip = os.path.join( + self.base_path, + PACKAGE_PATH_TEMPLATE % (project_id, function, md5sum) + ) if os.path.exists(func_zip): os.remove(func_zip) diff --git a/qinling/tests/unit/api/controllers/v1/test_execution.py b/qinling/tests/unit/api/controllers/v1/test_execution.py index 57f82013..48ee34df 100644 --- a/qinling/tests/unit/api/controllers/v1/test_execution.py +++ b/qinling/tests/unit/api/controllers/v1/test_execution.py @@ -18,14 +18,12 @@ from qinling import exceptions as exc from qinling import status from qinling.tests.unit.api import base -TEST_CASE_NAME = 'TestExecutionController' - class TestExecutionController(base.APITest): def setUp(self): super(TestExecutionController, self).setUp() - db_func = self.create_function(prefix=TEST_CASE_NAME) + db_func = self.create_function() self.func_id = db_func.id @mock.patch('qinling.rpc.EngineClient.create_execution') diff --git a/qinling/tests/unit/api/controllers/v1/test_function.py b/qinling/tests/unit/api/controllers/v1/test_function.py index 1ece42d9..a78f3a65 100644 --- a/qinling/tests/unit/api/controllers/v1/test_function.py +++ b/qinling/tests/unit/api/controllers/v1/test_function.py @@ -23,8 +23,6 @@ from qinling.tests.unit.api import base from qinling.tests.unit import base as unit_base from qinling.utils import constants -TEST_CASE_NAME = 'TestFunctionController' - class TestFunctionController(base.APITest): def setUp(self): @@ -32,7 +30,7 @@ class TestFunctionController(base.APITest): # Insert a runtime record in db for each test case. The data will be # removed automatically in tear down. - db_runtime = self.create_runtime(prefix=TEST_CASE_NAME) + db_runtime = self.create_runtime() self.runtime_id = db_runtime.id @mock.patch('qinling.storage.file_system.FileSystemStorage.store') @@ -41,7 +39,7 @@ class TestFunctionController(base.APITest): with tempfile.NamedTemporaryFile() as f: body = { - 'name': self.rand_name('function', prefix=TEST_CASE_NAME), + 'name': self.rand_name('function', prefix=self.prefix), 'code': json.dumps({"source": "package"}), 'runtime_id': self.runtime_id, } @@ -92,9 +90,7 @@ class TestFunctionController(base.APITest): self.assertEqual(400, resp.status_int) def test_get(self): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) expected = { 'id': db_func.id, "code": {"source": "package", "md5sum": "fake_md5"}, @@ -109,9 +105,7 @@ class TestFunctionController(base.APITest): self._assertDictContainsSubset(resp.json, expected) def test_get_all(self): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) expected = { 'id': db_func.id, "name": db_func.name, @@ -128,9 +122,7 @@ class TestFunctionController(base.APITest): self._assertDictContainsSubset(actual, expected) def test_put_name(self): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) resp = self.app.put_json( '/v1/functions/%s' % db_func.id, {'name': 'new_name'} @@ -145,9 +137,7 @@ class TestFunctionController(base.APITest): @mock.patch('qinling.rpc.EngineClient.delete_function') def test_put_package(self, mock_delete_func, mock_delete, mock_store, mock_etcd_del): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) mock_store.return_value = "fake_md5_changed" with tempfile.NamedTemporaryFile() as f: @@ -167,9 +157,7 @@ class TestFunctionController(base.APITest): db_func.id, "fake_md5") def test_put_package_same_md5(self): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) with tempfile.NamedTemporaryFile() as f: resp = self.app.put( @@ -189,9 +177,7 @@ class TestFunctionController(base.APITest): @mock.patch('qinling.rpc.EngineClient.delete_function') @mock.patch('qinling.storage.file_system.FileSystemStorage.delete') def test_delete(self, mock_delete, mock_delete_func, mock_etcd_delete): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) resp = self.app.delete('/v1/functions/%s' % db_func.id) self.assertEqual(204, resp.status_int) @@ -202,12 +188,9 @@ class TestFunctionController(base.APITest): mock_etcd_delete.assert_called_once_with(db_func.id) def test_delete_with_running_job(self): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) self.create_job( function_id=db_func.id, - prefix=TEST_CASE_NAME, status=status.AVAILABLE, first_execution_time=datetime.utcnow(), next_execution_time=datetime.utcnow(), @@ -222,10 +205,8 @@ class TestFunctionController(base.APITest): self.assertEqual(403, resp.status_int) def test_delete_with_webhook(self): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) - self.create_webhook(function_id=db_func.id, prefix=TEST_CASE_NAME) + db_func = self.create_function(runtime_id=self.runtime_id) + self.create_webhook(function_id=db_func.id) resp = self.app.delete( '/v1/functions/%s' % db_func.id, @@ -236,9 +217,7 @@ class TestFunctionController(base.APITest): @mock.patch('qinling.rpc.EngineClient.scaleup_function') def test_scale_up(self, scaleup_function_mock): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) body = {'count': 1} resp = self.app.post( @@ -254,9 +233,7 @@ class TestFunctionController(base.APITest): @mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.rpc.EngineClient.scaledown_function') def test_scale_down(self, scaledown_function_mock, get_workers_mock): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) get_workers_mock.return_value = [mock.Mock(), mock.Mock()] body = {'count': 1} @@ -274,9 +251,7 @@ class TestFunctionController(base.APITest): def test_scale_down_no_need( self, scaledown_function_mock, get_workers_mock ): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) get_workers_mock.return_value = [mock.Mock()] body = {'count': 1} @@ -294,9 +269,7 @@ class TestFunctionController(base.APITest): def test_detach( self, engine_delete_function_mock, etcd_delete_function_mock ): - db_func = self.create_function( - runtime_id=self.runtime_id, prefix=TEST_CASE_NAME - ) + db_func = self.create_function(runtime_id=self.runtime_id) resp = self.app.post( '/v1/functions/%s/detach' % db_func.id diff --git a/qinling/tests/unit/api/controllers/v1/test_function_version.py b/qinling/tests/unit/api/controllers/v1/test_function_version.py index b2e12c4d..8818a61b 100644 --- a/qinling/tests/unit/api/controllers/v1/test_function_version.py +++ b/qinling/tests/unit/api/controllers/v1/test_function_version.py @@ -12,21 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime +from datetime import timedelta + import mock from qinling import context from qinling.db import api as db_api +from qinling import status from qinling.tests.unit.api import base from qinling.tests.unit import base as unit_base -TESTCASE_NAME = 'TestFunctionVersionController' - class TestFunctionVersionController(base.APITest): def setUp(self): super(TestFunctionVersionController, self).setUp() - db_func = self.create_function(prefix=TESTCASE_NAME) + db_func = self.create_function() self.func_id = db_func.id @mock.patch('qinling.storage.file_system.FileSystemStorage.copy') @@ -117,3 +119,52 @@ class TestFunctionVersionController(base.APITest): self.assertEqual(200, resp.status_int) self.assertEqual("version 1", resp.json.get('description')) + + @mock.patch('qinling.storage.file_system.FileSystemStorage.delete') + def test_delete(self, mock_delete): + db_api.increase_function_version(self.func_id, 0, + description="version 1") + + resp = self.app.delete('/v1/functions/%s/versions/1' % self.func_id) + + self.assertEqual(204, resp.status_int) + mock_delete.assert_called_once_with(unit_base.DEFAULT_PROJECT_ID, + self.func_id, None, version=1) + + # We need to set context as it was removed after the API call + context.set_ctx(self.ctx) + + with db_api.transaction(): + func_db = db_api.get_function(self.func_id) + self.assertEqual(0, len(func_db.versions)) + self.assertEqual(0, func_db.latest_version) + + def test_delete_with_running_job(self): + db_api.increase_function_version(self.func_id, 0, + description="version 1") + self.create_job( + self.func_id, + function_version=1, + status=status.RUNNING, + first_execution_time=datetime.utcnow(), + next_execution_time=datetime.utcnow() + timedelta(hours=1), + ) + + resp = self.app.delete( + '/v1/functions/%s/versions/1' % self.func_id, + expect_errors=True + ) + + self.assertEqual(403, resp.status_int) + + def test_delete_with_webhook(self): + db_api.increase_function_version(self.func_id, 0, + description="version 1") + self.create_webhook(self.func_id, function_version=1) + + resp = self.app.delete( + '/v1/functions/%s/versions/1' % self.func_id, + expect_errors=True + ) + + self.assertEqual(403, resp.status_int) diff --git a/qinling/tests/unit/api/controllers/v1/test_function_worker.py b/qinling/tests/unit/api/controllers/v1/test_function_worker.py index 5f97b6bf..694d31f2 100644 --- a/qinling/tests/unit/api/controllers/v1/test_function_worker.py +++ b/qinling/tests/unit/api/controllers/v1/test_function_worker.py @@ -17,8 +17,6 @@ import mock from oslo_utils import uuidutils from qinling.tests.unit.api import base -TEST_CASE_NAME = 'TestFunctionWorkerController' - class TestFunctionWorkerController(base.APITest): @mock.patch('qinling.utils.etcd_util.get_workers') diff --git a/qinling/tests/unit/api/controllers/v1/test_job.py b/qinling/tests/unit/api/controllers/v1/test_job.py index 55e17ddb..6ca26fe4 100644 --- a/qinling/tests/unit/api/controllers/v1/test_job.py +++ b/qinling/tests/unit/api/controllers/v1/test_job.py @@ -27,12 +27,12 @@ class TestJobController(base.APITest): # Insert a function record in db for each test case. The data will be # removed automatically in db clean up. - db_function = self.create_function(prefix='TestJobController') + db_function = self.create_function() self.function_id = db_function.id def test_post(self): body = { - 'name': self.rand_name('job', prefix='TestJobController'), + 'name': self.rand_name('job', prefix=self.prefix), 'first_execution_time': str( datetime.utcnow() + timedelta(hours=1)), 'function_id': self.function_id @@ -43,7 +43,7 @@ class TestJobController(base.APITest): def test_post_pattern(self): body = { - 'name': self.rand_name('job', prefix='TestJobController'), + 'name': self.rand_name('job', prefix=self.prefix), 'function_id': self.function_id, 'pattern': '0 21 * * *', 'count': 10 @@ -54,7 +54,7 @@ class TestJobController(base.APITest): def test_delete(self): job_id = self.create_job( - self.function_id, prefix='TestJobController', + self.function_id, first_execution_time=datetime.utcnow(), next_execution_time=datetime.utcnow() + timedelta(hours=1), status=status.RUNNING, @@ -68,7 +68,6 @@ class TestJobController(base.APITest): def test_update_one_shot_job(self): job_id = self.create_job( self.function_id, - prefix='TestJobController', first_execution_time=datetime.utcnow(), next_execution_time=datetime.utcnow() + timedelta(hours=1), status=status.RUNNING, @@ -95,7 +94,6 @@ class TestJobController(base.APITest): def test_update_one_shot_job_failed(self): job_id = self.create_job( self.function_id, - prefix='TestJobController', first_execution_time=datetime.utcnow(), next_execution_time=datetime.utcnow() + timedelta(hours=1), status=status.RUNNING, @@ -138,7 +136,6 @@ class TestJobController(base.APITest): def test_update_recurring_job(self): job_id = self.create_job( self.function_id, - prefix='TestJobController', first_execution_time=datetime.utcnow() + timedelta(hours=1), next_execution_time=datetime.utcnow() + timedelta(hours=1), pattern='0 */1 * * *', @@ -195,7 +192,6 @@ class TestJobController(base.APITest): def test_update_recurring_job_failed(self): job_id = self.create_job( self.function_id, - prefix='TestJobController', first_execution_time=datetime.utcnow() + timedelta(hours=1), next_execution_time=datetime.utcnow() + timedelta(hours=1), pattern='0 */1 * * *', diff --git a/qinling/tests/unit/api/controllers/v1/test_runtime.py b/qinling/tests/unit/api/controllers/v1/test_runtime.py index 21cabf70..f225d3bc 100644 --- a/qinling/tests/unit/api/controllers/v1/test_runtime.py +++ b/qinling/tests/unit/api/controllers/v1/test_runtime.py @@ -26,7 +26,7 @@ class TestRuntimeController(base.APITest): # Insert a runtime record in db. The data will be removed in db clean # up. - self.db_runtime = self.create_runtime(prefix='TestRuntimeController') + self.db_runtime = self.create_runtime() self.runtime_id = self.db_runtime.id def test_get(self): @@ -65,8 +65,8 @@ class TestRuntimeController(base.APITest): @mock.patch('qinling.rpc.EngineClient.create_runtime') def test_post(self, mock_create_time): body = { - 'name': self.rand_name('runtime', prefix='TestRuntimeController'), - 'image': self.rand_name('image', prefix='TestRuntimeController'), + 'name': self.rand_name('runtime', prefix=self.prefix), + 'image': self.rand_name('image', prefix=self.prefix), } resp = self.app.post_json('/v1/runtimes', body) @@ -77,7 +77,7 @@ class TestRuntimeController(base.APITest): @mock.patch('qinling.rpc.EngineClient.create_runtime') def test_post_without_image(self, mock_create_time): body = { - 'name': self.rand_name('runtime', prefix='TestRuntimeController'), + 'name': self.rand_name('runtime', prefix=self.prefix), } resp = self.app.post_json('/v1/runtimes', body, expect_errors=True) @@ -94,7 +94,7 @@ class TestRuntimeController(base.APITest): @mock.patch('qinling.rpc.EngineClient.delete_runtime') def test_delete_runtime_with_function_associated(self, mock_delete_runtime): - self.create_function(self.runtime_id, prefix='TestRuntimeController') + self.create_function(self.runtime_id) resp = self.app.delete( '/v1/runtimes/%s' % self.runtime_id, expect_errors=True ) @@ -113,10 +113,8 @@ class TestRuntimeController(base.APITest): def test_put_image_runtime_not_available(self): db_runtime = db_api.create_runtime( { - 'name': self.rand_name( - 'runtime', prefix='TestRuntimeController'), - 'image': self.rand_name( - 'image', prefix='TestRuntimeController'), + 'name': self.rand_name('runtime', prefix=self.prefix), + 'image': self.rand_name('image', prefix=self.prefix), 'project_id': test_base.DEFAULT_PROJECT_ID, 'status': status.CREATING } @@ -148,8 +146,7 @@ class TestRuntimeController(base.APITest): @mock.patch('qinling.rpc.EngineClient.update_runtime') def test_put_image_not_allowed(self, mock_update_runtime, mock_etcd_url): mock_etcd_url.return_value = True - function_id = self.create_function( - self.runtime_id, prefix='TestRuntimeController').id + function_id = self.create_function(self.runtime_id).id resp = self.app.put_json( '/v1/runtimes/%s' % self.runtime_id, {'image': 'new_image'}, diff --git a/qinling/tests/unit/api/controllers/v1/test_webhook.py b/qinling/tests/unit/api/controllers/v1/test_webhook.py index 2c828e76..543918f6 100644 --- a/qinling/tests/unit/api/controllers/v1/test_webhook.py +++ b/qinling/tests/unit/api/controllers/v1/test_webhook.py @@ -14,13 +14,11 @@ from qinling.tests.unit.api import base -TEST_CASE_NAME = 'TestWebhookController' - class TestWebhookController(base.APITest): def setUp(self): super(TestWebhookController, self).setUp() - db_func = self.create_function(prefix=TEST_CASE_NAME) + db_func = self.create_function() self.func_id = db_func.id def test_crud(self): diff --git a/qinling/tests/unit/base.py b/qinling/tests/unit/base.py index 5f8a82cb..2747fdd9 100644 --- a/qinling/tests/unit/base.py +++ b/qinling/tests/unit/base.py @@ -111,6 +111,8 @@ class DbTestCase(BaseTest): def setUp(self): super(DbTestCase, self).setUp() + self.prefix = self.__class__.__name__ + self._heavy_init() self.ctx = get_context() @@ -164,11 +166,11 @@ class DbTestCase(BaseTest): def _clean_db(self): db_api.delete_all() - def create_runtime(self, prefix=None): + def create_runtime(self): runtime = db_api.create_runtime( { - 'name': self.rand_name('runtime', prefix=prefix), - 'image': self.rand_name('image', prefix=prefix), + 'name': self.rand_name('runtime', prefix=self.prefix), + 'image': self.rand_name('image', prefix=self.prefix), # 'auth_enable' is disabled by default, we create runtime for # default tenant. 'project_id': DEFAULT_PROJECT_ID, @@ -178,13 +180,13 @@ class DbTestCase(BaseTest): return runtime - def create_function(self, runtime_id=None, prefix=None): + def create_function(self, runtime_id=None): if not runtime_id: - runtime_id = self.create_runtime(prefix).id + runtime_id = self.create_runtime().id function = db_api.create_function( { - 'name': self.rand_name('function', prefix=prefix), + 'name': self.rand_name('function', prefix=self.prefix), 'runtime_id': runtime_id, 'code': {"source": "package", "md5sum": "fake_md5"}, 'entry': 'main.main', @@ -196,12 +198,12 @@ class DbTestCase(BaseTest): return function - def create_job(self, function_id=None, prefix=None, **kwargs): + def create_job(self, function_id=None, **kwargs): if not function_id: - function_id = self.create_function(prefix=prefix).id + function_id = self.create_function().id job_params = { - 'name': self.rand_name('job', prefix=prefix), + 'name': self.rand_name('job', prefix=self.prefix), 'function_id': function_id, # 'auth_enable' is disabled by default 'project_id': DEFAULT_PROJECT_ID, @@ -211,9 +213,9 @@ class DbTestCase(BaseTest): return job - def create_webhook(self, function_id=None, prefix=None, **kwargs): + def create_webhook(self, function_id=None, **kwargs): if not function_id: - function_id = self.create_function(prefix=prefix).id + function_id = self.create_function().id webhook_params = { 'function_id': function_id, @@ -225,9 +227,9 @@ class DbTestCase(BaseTest): return webhook - def create_execution(self, function_id=None, prefix=None, **kwargs): + def create_execution(self, function_id=None, **kwargs): if not function_id: - function_id = self.create_function(prefix=prefix).id + function_id = self.create_function().id execution_params = { 'function_id': function_id, @@ -239,10 +241,9 @@ class DbTestCase(BaseTest): return execution - def create_function_version(self, old_version, function_id=None, - prefix=None, **kwargs): + def create_function_version(self, old_version, function_id=None, **kwargs): if not function_id: - function_id = self.create_function(prefix=prefix).id + function_id = self.create_function().id db_api.increase_function_version(function_id, old_version, **kwargs) db_api.update_function(function_id, diff --git a/qinling/tests/unit/engine/test_default_engine.py b/qinling/tests/unit/engine/test_default_engine.py index 8ca72956..f5a7e781 100644 --- a/qinling/tests/unit/engine/test_default_engine.py +++ b/qinling/tests/unit/engine/test_default_engine.py @@ -33,11 +33,10 @@ class TestDefaultEngine(base.DbTestCase): def _create_running_executions(self, function_id, num): for _i in range(num): - self.create_execution(function_id=function_id, - prefix='TestDefaultEngine') + self.create_execution(function_id=function_id) def test_create_runtime(self): - runtime = self.create_runtime(prefix='TestDefaultEngine') + runtime = self.create_runtime() runtime_id = runtime.id # Set status to verify it is changed during creation. db_api.update_runtime(runtime_id, {'status': status.CREATING}) @@ -50,7 +49,7 @@ class TestDefaultEngine(base.DbTestCase): self.assertEqual(status.AVAILABLE, runtime.status) def test_create_runtime_failed(self): - runtime = self.create_runtime(prefix='TestDefaultEngine') + runtime = self.create_runtime() runtime_id = runtime.id # Set status to verify it is changed during creation. db_api.update_runtime(runtime_id, {'status': status.CREATING}) @@ -64,7 +63,7 @@ class TestDefaultEngine(base.DbTestCase): self.assertEqual(status.ERROR, runtime.status) def test_delete_runtime(self): - runtime = self.create_runtime(prefix='TestDefaultEngine') + runtime = self.create_runtime() runtime_id = runtime.id self.default_engine.delete_runtime(mock.Mock(), runtime_id) @@ -77,12 +76,12 @@ class TestDefaultEngine(base.DbTestCase): db_api.get_runtime, runtime_id) def test_update_runtime(self): - runtime = self.create_runtime(prefix='TestDefaultEngine') + runtime = self.create_runtime() runtime_id = runtime.id # Set status to verify it is changed during update. db_api.update_runtime(runtime_id, {'status': status.UPGRADING}) - image = self.rand_name('new_image', prefix='TestDefaultEngine') - pre_image = self.rand_name('pre_image', prefix='TestDefaultEngine') + image = self.rand_name('new_image', prefix=self.prefix) + pre_image = self.rand_name('pre_image', prefix=self.prefix) self.orchestrator.update_pool.return_value = True self.default_engine.update_runtime( @@ -94,12 +93,12 @@ class TestDefaultEngine(base.DbTestCase): self.assertEqual(runtime.status, status.AVAILABLE) def test_update_runtime_rollbacked(self): - runtime = self.create_runtime(prefix='TestDefaultEngine') + runtime = self.create_runtime() runtime_id = runtime.id # Set status to verify it is changed during update. db_api.update_runtime(runtime_id, {'status': status.UPGRADING}) - image = self.rand_name('new_image', prefix='TestDefaultEngine') - pre_image = self.rand_name('pre_image', prefix='TestDefaultEngine') + image = self.rand_name('new_image', prefix=self.prefix) + pre_image = self.rand_name('pre_image', prefix=self.prefix) self.orchestrator.update_pool.return_value = False self.default_engine.update_runtime( @@ -141,7 +140,7 @@ class TestDefaultEngine(base.DbTestCase): etcd_util_get_worker_lock_mock, etcd_util_get_workers_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id lock = mock.Mock() @@ -168,7 +167,7 @@ class TestDefaultEngine(base.DbTestCase): etcd_util_get_worker_lock_mock, etcd_util_get_workers_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id lock = mock.Mock() @@ -194,7 +193,7 @@ class TestDefaultEngine(base.DbTestCase): etcd_util_get_worker_lock_mock, etcd_util_get_workers_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id lock = mock.Mock() @@ -218,7 +217,7 @@ class TestDefaultEngine(base.DbTestCase): self, etcd_util_get_service_url_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id db_api.update_function( @@ -226,17 +225,14 @@ class TestDefaultEngine(base.DbTestCase): { 'code': { 'source': constants.IMAGE_FUNCTION, - 'image': self.rand_name('image', - prefix='TestDefaultEngine') + 'image': self.rand_name('image', prefix=self.prefix) } } ) function = db_api.get_function(function_id) - execution_1 = self.create_execution( - function_id=function_id, prefix='TestDefaultEngine') + execution_1 = self.create_execution(function_id=function_id) execution_1_id = execution_1.id - execution_2 = self.create_execution( - function_id=function_id, prefix='TestDefaultEngine') + execution_2 = self.create_execution(function_id=function_id) execution_2_id = execution_2.id self.default_engine.function_load_check = mock.Mock() etcd_util_get_service_url_mock.return_value = None @@ -302,7 +298,7 @@ class TestDefaultEngine(base.DbTestCase): self, etcd_util_get_service_url_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id db_api.update_function( @@ -310,14 +306,12 @@ class TestDefaultEngine(base.DbTestCase): { 'code': { 'source': constants.IMAGE_FUNCTION, - 'image': self.rand_name('image', - prefix='TestDefaultEngine') + 'image': self.rand_name('image', prefix=self.prefix) } } ) function = db_api.get_function(function_id) - execution = self.create_execution( - function_id=function_id, prefix='TestDefaultEngine') + execution = self.create_execution(function_id=function_id) execution_id = execution.id prepare_execution = self.orchestrator.prepare_execution prepare_execution.side_effect = exc.OrchestratorException( @@ -338,11 +332,10 @@ class TestDefaultEngine(base.DbTestCase): self, etcd_util_get_service_url_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id - execution = self.create_execution( - function_id=function_id, prefix='TestDefaultEngine') + execution = self.create_execution(function_id=function_id) execution_id = execution.id self.default_engine.function_load_check = mock.Mock(return_value='') etcd_util_get_service_url_mock.return_value = None @@ -372,11 +365,10 @@ class TestDefaultEngine(base.DbTestCase): self.assertEqual(execution.result, {'output': 'success output'}) def test_create_execution_not_image_source_scaleup_exception(self): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id - execution = self.create_execution( - function_id=function_id, prefix='TestDefaultEngine') + execution = self.create_execution(function_id=function_id) execution_id = execution.id self.default_engine.function_load_check = mock.Mock( side_effect=exc.OrchestratorException( @@ -401,11 +393,10 @@ class TestDefaultEngine(base.DbTestCase): engine_utils_url_request_mock, engine_utils_get_request_data_mock ): - function = self.create_function(prefix='TestDefaultEngine') + function = self.create_function() function_id = function.id runtime_id = function.runtime_id - execution = self.create_execution( - function_id=function_id, prefix='TestDefaultEngine') + execution = self.create_execution(function_id=function_id) execution_id = execution.id self.default_engine.function_load_check = mock.Mock(return_value='') etcd_util_get_service_url_mock.return_value = 'svc_url' diff --git a/qinling/tests/unit/services/test_periodics.py b/qinling/tests/unit/services/test_periodics.py index b848d26a..ee1893ba 100644 --- a/qinling/tests/unit/services/test_periodics.py +++ b/qinling/tests/unit/services/test_periodics.py @@ -29,8 +29,6 @@ CONF = cfg.CONF class TestPeriodics(base.DbTestCase): - TEST_CASE_NAME = 'TestPeriodics' - def setUp(self): super(TestPeriodics, self).setUp() self.override_config('auth_enable', False, group='pecan') @@ -39,9 +37,7 @@ class TestPeriodics(base.DbTestCase): @mock.patch('qinling.utils.etcd_util.get_service_url') def test_function_service_expiration_handler(self, mock_etcd_url, mock_etcd_delete): - db_func = self.create_function( - runtime_id=None, prefix=self.TEST_CASE_NAME - ) + db_func = self.create_function() function_id = db_func.id # Update function to simulate function execution db_api.update_function(function_id, {'count': 1}) @@ -60,9 +56,7 @@ class TestPeriodics(base.DbTestCase): @mock.patch('qinling.utils.jobs.get_next_execution_time') def test_job_handler(self, mock_get_next): - db_func = self.create_function( - runtime_id=None, prefix=self.TEST_CASE_NAME - ) + db_func = self.create_function() function_id = db_func.id self.assertEqual(0, db_func.count) @@ -70,7 +64,6 @@ class TestPeriodics(base.DbTestCase): now = datetime.utcnow() db_job = self.create_job( function_id, - self.TEST_CASE_NAME, status=status.RUNNING, next_execution_time=now, count=2 diff --git a/qinling/tests/unit/storage/test_file_system.py b/qinling/tests/unit/storage/test_file_system.py index 5e39ff82..eb9f8080 100644 --- a/qinling/tests/unit/storage/test_file_system.py +++ b/qinling/tests/unit/storage/test_file_system.py @@ -199,6 +199,25 @@ class TestFileSystemStorage(base.BaseTest): exists_mock.assert_called_once_with(package_path) remove_mock.assert_called_once_with(package_path) + @mock.patch('os.path.exists') + @mock.patch('os.remove') + @mock.patch('os.listdir') + def test_delete_with_version(self, mock_list, remove_mock, exists_mock): + exists_mock.return_value = True + function = self.rand_name('function', prefix='TestFileSystemStorage') + version = 1 + mock_list.return_value = ["%s_%s_md5.zip" % (function, version)] + + self.storage.delete(self.project_id, function, "fake_md5", version=1) + + package_path = os.path.join( + FAKE_STORAGE_PATH, + self.project_id, + file_system.PACKAGE_VERSION_TEMPLATE % (function, version, "md5") + ) + exists_mock.assert_called_once_with(package_path) + remove_mock.assert_called_once_with(package_path) + @mock.patch('os.path.exists') @mock.patch('os.remove') def test_delete_package_not_exists(self, remove_mock, exists_mock):