From d552edc8550ad6413a423d0bebf6d9ffa2f77fd7 Mon Sep 17 00:00:00 2001 From: Victor Ryzhenkin Date: Wed, 15 Jul 2015 16:47:37 +0300 Subject: [PATCH] Perform tests tagging This feature allows us to run a specifuc set of tests in different jobs on Murano CI. Change-Id: Ieda816b0f9efc8ed8320b1bc42b756a7bf713d70 Implements: blueprint new-ci-tests-preparation --- .../functional/api/v1/test_env_templates.py | 14 +++++++++++++ murano/tests/functional/api/v1/test_envs.py | 12 +++++++++++ .../functional/api/v1/test_repository.py | 21 +++++++++++++++++++ .../tests/functional/api/v1/test_services.py | 18 ++++++++++++++++ .../tests/functional/api/v1/test_sessions.py | 13 ++++++++++++ .../functional/engine/test_deployment.py | 3 +++ .../tests/functional/engine/test_mistral.py | 3 +++ .../functional/engine/test_policy_enf.py | 4 ++++ 8 files changed, 88 insertions(+) diff --git a/murano/tests/functional/api/v1/test_env_templates.py b/murano/tests/functional/api/v1/test_env_templates.py index 243a5b5d..eb4e6379 100644 --- a/murano/tests/functional/api/v1/test_env_templates.py +++ b/murano/tests/functional/api/v1/test_env_templates.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nose.plugins.attrib import attr as tag from tempest.test import attr from tempest_lib import exceptions @@ -20,6 +21,7 @@ from murano.tests.functional.api import base class TestEnvTemplate(base.TestCase): + @tag('all', 'coverage') @attr(type='smoke') def test_list_env_templates(self): """Check getting the list of environment templates.""" @@ -28,6 +30,7 @@ class TestEnvTemplate(base.TestCase): self.assertIn('templates', body) self.assertEqual(resp.status, 200) + @tag('all', 'coverage') @attr(type='smoke') def test_create_and_delete_env_template(self): """It checks the creation and deletion of an enviroment template.""" @@ -53,6 +56,7 @@ class TestEnvTemplate(base.TestCase): self.env_templates.pop(self.env_templates.index(env_template)) + @tag('all', 'coverage') @attr(type='smoke') def test_get_env_template(self): """Check getting information about an environment template.""" @@ -65,6 +69,7 @@ class TestEnvTemplate(base.TestCase): self.assertEqual(env_obtained_template['name'], 'test_env_temp') self.client.delete_env_template(env_template['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_create_env_template_with_apps(self): """Check the creation of an environment template with applications.""" @@ -77,6 +82,7 @@ class TestEnvTemplate(base.TestCase): self.assertEqual(len(apps_template), 1) self.client.delete_env_template(env_template['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_create_app_in_env_template(self): """Check the creationg of applications in an environment template.""" @@ -93,6 +99,7 @@ class TestEnvTemplate(base.TestCase): self.client.delete_env_template(env_template['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_delete_app_in_env_template(self): """Check the deletion of applications in an environmente template.""" @@ -111,6 +118,7 @@ class TestEnvTemplate(base.TestCase): self.client.delete_env_template(env_template['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_create_env_from_template(self): """Check the creation of an environment from a template.""" @@ -125,6 +133,7 @@ class TestEnvTemplate(base.TestCase): self.client.delete_env_template(env_template['id']) self.client.delete_environment(env['environment_id']) + @tag('all', 'coverage') @attr(type='negative') def test_delete_environment_with_wrong_env_id(self): """Check the deletion of an wrong environment template request.""" @@ -132,6 +141,7 @@ class TestEnvTemplate(base.TestCase): self.client.delete_env_template, None) + @tag('all', 'coverage') @attr(type='negative') def test_create_environment_with_wrong_payload(self): """Check the deletion of an wrong environment template request.""" @@ -139,6 +149,7 @@ class TestEnvTemplate(base.TestCase): self.client.create_env_template, '-+3') + @tag('all', 'coverage') @attr(type='negative') def test_double_delete_env_template(self): """Check the deletion of an wrong environment template request.""" @@ -150,6 +161,7 @@ class TestEnvTemplate(base.TestCase): self.client.delete_env_template, env_template['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_deleted_env_template(self): """Check the deletion of an wrong environment template request.""" @@ -164,6 +176,7 @@ class TestEnvTemplate(base.TestCase): class TestEnvTemplatesTenantIsolation(base.NegativeTestCase): + @tag('all', 'coverage') @attr(type='negative') def test_get_env_template_from_another_tenant(self): """It tests getting information from an environment @@ -176,6 +189,7 @@ class TestEnvTemplatesTenantIsolation(base.NegativeTestCase): self.client.delete_env_template(env_template['id']) + @tag('all', 'coverage') @attr(type='negative') def test_delete_env_template_from_another_tenant(self): """It tests deleting information from an environment diff --git a/murano/tests/functional/api/v1/test_envs.py b/murano/tests/functional/api/v1/test_envs.py index 7f8ad7a4..94298dfa 100644 --- a/murano/tests/functional/api/v1/test_envs.py +++ b/murano/tests/functional/api/v1/test_envs.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nose.plugins.attrib import attr as tag from tempest.test import attr from tempest_lib import exceptions @@ -20,6 +21,7 @@ from murano.tests.functional.api import base class TestEnvironments(base.TestCase): + @tag('all', 'coverage') @attr(type='smoke') def test_list_environments(self): resp, body = self.client.get_environments_list() @@ -27,6 +29,7 @@ class TestEnvironments(base.TestCase): self.assertIn('environments', body) self.assertEqual(resp.status, 200) + @tag('all', 'coverage') @attr(type='smoke') def test_create_and_delete_environment(self): environments_list_start = self.client.get_environments_list()[1] @@ -51,6 +54,7 @@ class TestEnvironments(base.TestCase): self.environments.pop(self.environments.index(env)) + @tag('all', 'coverage') @attr(type='smoke') def test_get_environment(self): env = self.create_environment('test') @@ -60,6 +64,7 @@ class TestEnvironments(base.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(environment['name'], 'test') + @tag('all', 'coverage') @attr(type='smoke') def test_update_environment(self): env = self.create_environment('test') @@ -69,18 +74,21 @@ class TestEnvironments(base.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(environment['name'], 'changed-environment-name') + @tag('all', 'coverage') @attr(type='negative') def test_update_environment_with_wrong_env_id(self): self.assertRaises(exceptions.NotFound, self.client.update_environment, None) + @tag('all', 'coverage') @attr(type='negative') def test_delete_environment_with_wrong_env_id(self): self.assertRaises(exceptions.NotFound, self.client.delete_environment, None) + @tag('all', 'coverage') @attr(type='negative') def test_double_delete_environment(self): env = self.create_environment('test') @@ -91,6 +99,7 @@ class TestEnvironments(base.TestCase): self.client.delete_environment, env['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_deleted_environment(self): env = self.create_environment('test') @@ -104,6 +113,7 @@ class TestEnvironments(base.TestCase): class TestEnvironmentsTenantIsolation(base.NegativeTestCase): + @tag('all', 'coverage') @attr(type='negative') def test_get_environment_from_another_tenant(self): env = self.create_environment('test') @@ -111,6 +121,7 @@ class TestEnvironmentsTenantIsolation(base.NegativeTestCase): self.assertRaises(exceptions.Forbidden, self.alt_client.get_environment, env['id']) + @tag('all', 'coverage') @attr(type='negative') def test_update_environment_from_another_tenant(self): env = self.create_environment('test') @@ -118,6 +129,7 @@ class TestEnvironmentsTenantIsolation(base.NegativeTestCase): self.assertRaises(exceptions.Forbidden, self.alt_client.update_environment, env['id']) + @tag('all', 'coverage') @attr(type='negative') def test_delete_environment_from_another_tenant(self): env = self.create_environment('test') diff --git a/murano/tests/functional/api/v1/test_repository.py b/murano/tests/functional/api/v1/test_repository.py index c1d56f21..ce2a5907 100644 --- a/murano/tests/functional/api/v1/test_repository.py +++ b/murano/tests/functional/api/v1/test_repository.py @@ -15,6 +15,7 @@ import os import uuid +from nose.plugins.attrib import attr as tag from tempest.test import attr from tempest_lib import exceptions @@ -58,6 +59,7 @@ class TestCaseRepository(base.TestCase, common_utils.ZipUtilsMixin): class TestRepositorySanity(TestCaseRepository): + @tag('all', 'coverage') @attr(type='smoke') def test_get_list_packages(self): resp, body = self.client.get_list_packages() @@ -65,6 +67,7 @@ class TestRepositorySanity(TestCaseRepository): self.assertEqual(200, resp.status) self.assertTrue(isinstance(body['packages'], list)) + @tag('all', 'coverage') @attr(type='smoke') def test_get_list_categories(self): resp, body = self.client.list_categories() @@ -72,6 +75,7 @@ class TestRepositorySanity(TestCaseRepository): self.assertEqual(200, resp.status) self.assertTrue(isinstance(body['categories'], list)) + @tag('all', 'coverage') @attr(type='smoke') def test_upload_and_delete_package(self): packages_list = self.client.get_list_packages()[1] @@ -107,6 +111,7 @@ class TestRepositoryNegativeNotFound(base.NegativeTestCase): cls.id = uuid.uuid4().hex + @tag('all', 'coverage') @attr(type='negative') def test_update_package_with_incorrect_id(self): @@ -123,30 +128,35 @@ class TestRepositoryNegativeNotFound(base.NegativeTestCase): self.id, post_body) + @tag('all', 'coverage') @attr(type='negative') def test_get_package_with_incorrect_id(self): self.assertRaises(exceptions.NotFound, self.client.get_package, self.id) + @tag('all', 'coverage') @attr(type='negative') def test_delete_package_with_incorrect_id(self): self.assertRaises(exceptions.NotFound, self.client.delete_package, self.id) + @tag('all', 'coverage') @attr(type='negative') def test_download_package_with_incorrect_id(self): self.assertRaises(exceptions.NotFound, self.client.download_package, self.id) + @tag('all', 'coverage') @attr(type='negative') def test_get_ui_definition_with_incorrect_id(self): self.assertRaises(exceptions.NotFound, self.client.get_ui_definition, self.id) + @tag('all', 'coverage') @attr(type='negative') def test_get_logo_with_incorrect_id(self): self.assertRaises(exceptions.NotFound, @@ -184,6 +194,7 @@ class TestRepositoryNegativeForbidden(base.NegativeTestCase, cls.client.delete_package(cls.package['id']) cls.purge_creds() + @tag('all', 'coverage') @attr(type='negative') def test_update_package_from_another_tenant(self): post_body = [ @@ -199,30 +210,35 @@ class TestRepositoryNegativeForbidden(base.NegativeTestCase, self.package['id'], post_body) + @tag('all', 'coverage') @attr(type='negative') def test_get_package_from_another_tenant(self): self.assertRaises(exceptions.Forbidden, self.alt_client.get_package, self.package['id']) + @tag('all', 'coverage') @attr(type='negative') def test_delete_package_from_another_tenant(self): self.assertRaises(exceptions.Forbidden, self.alt_client.delete_package, self.package['id']) + @tag('all', 'coverage') @attr(type='negative') def test_download_package_from_another_tenant(self): self.assertRaises(exceptions.Forbidden, self.alt_client.download_package, self.package['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_ui_definition_from_another_tenant(self): self.assertRaises(exceptions.Forbidden, self.alt_client.get_ui_definition, self.package['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_logo_from_another_tenant(self): self.assertRaises(exceptions.Forbidden, @@ -250,6 +266,7 @@ class TestRepository(TestCaseRepository): {"categories": [self.categorie], "tags": ["windows"]}).json() self.packages.append(self.package) + @tag('all', 'coverage') @attr(type='smoke') def test_get_package(self): resp, body = self.client.get_package(self.package['id']) @@ -257,6 +274,7 @@ class TestRepository(TestCaseRepository): self.assertEqual(200, resp.status) self.assertEqual(self.package['tags'], body['tags']) + @tag('all', 'coverage') @attr(type='smoke') def test_update_package(self): post_body = [ @@ -351,18 +369,21 @@ class TestRepository(TestCaseRepository): self.assertEqual(200, resp.status) self.assertEqual("New name", body['name']) + @tag('all', 'coverage') @attr(type='smoke') def test_download_package(self): resp = self.client.download_package(self.package['id'])[0] self.assertEqual(200, resp.status) + @tag('all', 'coverage') @attr(type='smoke') def test_get_ui_definitions(self): resp = self.client.get_ui_definition(self.package['id'])[0] self.assertEqual(200, resp.status) + @tag('all', 'coverage') @attr(type='smoke') def test_get_logo(self): resp, body = self.client.get_logo(self.package['id']) diff --git a/murano/tests/functional/api/v1/test_services.py b/murano/tests/functional/api/v1/test_services.py index 348c1d1e..08f2de51 100644 --- a/murano/tests/functional/api/v1/test_services.py +++ b/murano/tests/functional/api/v1/test_services.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nose.plugins.attrib import attr as tag from tempest.test import attr from tempest_lib import exceptions @@ -20,6 +21,7 @@ from murano.tests.functional.api import base class TestServices(base.TestCase): + @tag('all', 'coverage') @attr(type='smoke') def test_get_services_list(self): env = self.create_environment('test') @@ -32,6 +34,7 @@ class TestServices(base.TestCase): self.assertEqual(resp.status, 200) self.assertTrue(isinstance(services_list, list)) + @tag('all', 'coverage') @attr(type='negative') def test_get_services_list_without_env_id(self): env = self.create_environment('test') @@ -43,6 +46,7 @@ class TestServices(base.TestCase): None, sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_services_list_after_delete_env(self): env = self.create_environment('test') @@ -56,6 +60,7 @@ class TestServices(base.TestCase): env['id'], sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_services_list_after_delete_session(self): env = self.create_environment('test') @@ -69,6 +74,7 @@ class TestServices(base.TestCase): env['id'], sess['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_create_and_delete_demo_service(self): env = self.create_environment('test') @@ -95,6 +101,7 @@ class TestServices(base.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(len(services_list), len(services_list_)) + @tag('all', 'coverage') @attr(type='negative') def test_create_demo_service_without_env_id(self): env = self.create_environment('test') @@ -106,6 +113,7 @@ class TestServices(base.TestCase): None, sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_create_demo_service_without_sess_id(self): env = self.create_environment('test') @@ -117,6 +125,7 @@ class TestServices(base.TestCase): env['id'], "") + @tag('all', 'coverage') @attr(type='negative') def test_delete_demo_service_without_env_id(self): env = self.create_environment('test') @@ -131,6 +140,7 @@ class TestServices(base.TestCase): sess['id'], service['?']['id']) + @tag('all', 'coverage') @attr(type='negative') def test_delete_demo_service_without_session_id(self): env = self.create_environment('test') @@ -145,6 +155,7 @@ class TestServices(base.TestCase): "", service['?']['id']) + @tag('all', 'coverage') @attr(type='negative') def test_double_delete_service(self): env = self.create_environment('test') @@ -161,6 +172,7 @@ class TestServices(base.TestCase): sess['id'], service['?']['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_get_service(self): env = self.create_environment('test') @@ -176,6 +188,7 @@ class TestServices(base.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(service, service_) + @tag('all', 'coverage') @attr(type='negative') def test_get_service_without_env_id(self): env = self.create_environment('test') @@ -190,6 +203,7 @@ class TestServices(base.TestCase): sess['id'], service['?']['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_get_services_without_sess_id(self): env = self.create_environment('test') @@ -200,6 +214,7 @@ class TestServices(base.TestCase): class TestServicesTenantIsolation(base.NegativeTestCase): + @tag('all', 'coverage') @attr(type='negative') def test_get_list_services_in_env_from_another_tenant(self): env = self.create_environment('test') @@ -209,6 +224,7 @@ class TestServicesTenantIsolation(base.NegativeTestCase): self.alt_client.get_services_list, env['id'], sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_create_service_in_env_from_another_tenant(self): env = self.create_environment('test') @@ -218,6 +234,7 @@ class TestServicesTenantIsolation(base.NegativeTestCase): self.create_demo_service, env['id'], sess['id'], client=self.alt_client) + @tag('all', 'coverage') @attr(type='negative') def test_delete_service_in_env_from_another_tenant(self): env = self.create_environment('test') @@ -228,6 +245,7 @@ class TestServicesTenantIsolation(base.NegativeTestCase): self.alt_client.delete_service, env['id'], sess['id'], service['?']['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_service_in_env_from_another_tenant(self): env = self.create_environment('test') diff --git a/murano/tests/functional/api/v1/test_sessions.py b/murano/tests/functional/api/v1/test_sessions.py index 2c5347de..65d4ca25 100644 --- a/murano/tests/functional/api/v1/test_sessions.py +++ b/murano/tests/functional/api/v1/test_sessions.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nose.plugins.attrib import attr as tag from tempest.test import attr from tempest_lib import exceptions import testtools @@ -21,6 +22,7 @@ from murano.tests.functional.api import base class TestSessions(base.TestCase): + @tag('all', 'coverage') @attr(type='smoke') def test_create_session(self): env = self.create_environment('test') @@ -30,12 +32,14 @@ class TestSessions(base.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(env['id'], sess['environment_id']) + @tag('all', 'coverage') @attr(type='negative') def test_create_session_before_env(self): self.assertRaises(exceptions.NotFound, self.client.create_session, None) + @tag('all', 'coverage') @attr(type='smoke') def test_delete_session(self): env = self.create_environment('test') @@ -46,6 +50,7 @@ class TestSessions(base.TestCase): self.assertEqual(resp.status, 200) + @tag('all', 'coverage') @attr(type='negative') def test_delete_session_without_env_id(self): env = self.create_environment('test') @@ -57,6 +62,7 @@ class TestSessions(base.TestCase): None, sess['id']) + @tag('all', 'coverage') @attr(type='smoke') def test_get_session(self): env = self.create_environment('test') @@ -68,6 +74,7 @@ class TestSessions(base.TestCase): self.assertEqual(resp.status, 200) self.assertEqual(session, sess) + @tag('all', 'coverage') @attr(type='negative') def test_get_session_without_env_id(self): env = self.create_environment('test') @@ -79,6 +86,7 @@ class TestSessions(base.TestCase): None, sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_session_after_delete_env(self): env = self.create_environment('test') @@ -92,6 +100,7 @@ class TestSessions(base.TestCase): env['id'], sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_double_delete_session(self): env = self.create_environment('test') @@ -108,6 +117,7 @@ class TestSessions(base.TestCase): class TestSessionsTenantIsolation(base.NegativeTestCase): + @tag('all', 'coverage') @attr(type='negative') def test_create_session_in_env_from_another_tenant(self): env = self.create_environment('test') @@ -115,6 +125,7 @@ class TestSessionsTenantIsolation(base.NegativeTestCase): self.assertRaises(exceptions.Forbidden, self.alt_client.create_session, env['id']) + @tag('all', 'coverage') @attr(type='negative') def test_delete_session_in_env_from_another_tenant(self): env = self.create_environment('test') @@ -124,6 +135,7 @@ class TestSessionsTenantIsolation(base.NegativeTestCase): self.alt_client.delete_session, env['id'], sess['id']) + @tag('all', 'coverage') @attr(type='negative') def test_get_session_in_env_from_another_tenant(self): env = self.create_environment('test') @@ -133,6 +145,7 @@ class TestSessionsTenantIsolation(base.NegativeTestCase): self.alt_client.get_session, env['id'], sess['id']) + @tag('all', 'coverage') @testtools.skip("https://bugs.launchpad.net/murano/+bug/1382026") @attr(type='negative') def test_deploy_session_in_env_from_another_tenant(self): diff --git a/murano/tests/functional/engine/test_deployment.py b/murano/tests/functional/engine/test_deployment.py index a55d0f59..185254a0 100644 --- a/murano/tests/functional/engine/test_deployment.py +++ b/murano/tests/functional/engine/test_deployment.py @@ -14,6 +14,8 @@ import uuid +from nose.plugins.attrib import attr as tag + import murano.tests.functional.engine.manager as core @@ -37,6 +39,7 @@ class MuranoDeploymentTest(core.MuranoTestsCore): cls.purge_environments() cls.purge_uploaded_packages() + @tag('gate', 'all', 'coverage') def test_app_deployment(self): post_body = { "instance": { diff --git a/murano/tests/functional/engine/test_mistral.py b/murano/tests/functional/engine/test_mistral.py index d6920500..f1a26701 100644 --- a/murano/tests/functional/engine/test_mistral.py +++ b/murano/tests/functional/engine/test_mistral.py @@ -14,6 +14,8 @@ import uuid +from nose.plugins.attrib import attr as tag + import murano.tests.functional.common.utils as common_utils import murano.tests.functional.engine.integration_base as core @@ -39,6 +41,7 @@ class MistralTest(core.MistralIntegration): with common_utils.ignored(Exception): cls.purge_uploaded_packages() + @tag('all', 'coverage') def test_deploy_package_success(self): # Test expects successful deployment and one output: input_1_value. diff --git a/murano/tests/functional/engine/test_policy_enf.py b/murano/tests/functional/engine/test_policy_enf.py index 86f0f2a5..f01f10ee 100644 --- a/murano/tests/functional/engine/test_policy_enf.py +++ b/murano/tests/functional/engine/test_policy_enf.py @@ -14,6 +14,8 @@ import muranoclient.common.exceptions as murano_exceptions +from nose.plugins.attrib import attr as tag + import murano.tests.functional.common.utils as common_utils import murano.tests.functional.engine.integration_base as core @@ -38,6 +40,7 @@ class PolicyEnforcement(core.CongressIntegration): super(PolicyEnforcement, self).tearDown() self.purge_environments() + @tag('all', 'coverage') def test_deploy_policy_fail_flavor(self): """Test expects failure due to blacklisted flavor.""" @@ -46,6 +49,7 @@ class PolicyEnforcement(core.CongressIntegration): key="test-key"), "bad flavor") + @tag('all', 'coverage') def test_deploy_policy_fail_key(self): """Test expects failure due to empty key name."""