diff --git a/config_tempest/clients.py b/config_tempest/clients.py index 7b527140..155465d8 100644 --- a/config_tempest/clients.py +++ b/config_tempest/clients.py @@ -30,6 +30,7 @@ from tempest.lib.services.identity.v3 import services_client as s_client from tempest.lib.services.identity.v3 import users_client as users_v3_client from tempest.lib.services.image.v2 import images_client from tempest.lib.services.network import networks_client +from tempest.lib.services.object_storage import account_client try: # Since Rocky, volume.v3.services_client is the default from tempest.lib.services.volume.v3 import services_client @@ -119,6 +120,12 @@ class ClientManager(object): self.identity_region, **default_params) + self.accounts = account_client.AccountClient( + self.auth_provider, + conf.get_defaulted('object-storage', 'catalog_type'), + self.identity_region, + **default_params) + self.set_users_client( auth=self.auth_provider, identity_version=creds.identity_version, @@ -227,7 +234,7 @@ class ClientManager(object): # and so on. if service_name == "image": return self.images - elif service_name == "network": + elif service_name in ["network", "object-store"]: # return whole ClientManager object because NetworkService # currently needs to have an access to get_neutron/nova_client # methods which are chosen according to neutron presence diff --git a/config_tempest/services/object_storage.py b/config_tempest/services/object_storage.py index 277825c5..8a5575f2 100644 --- a/config_tempest/services/object_storage.py +++ b/config_tempest/services/object_storage.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import ConfigParser import json from base import Service @@ -21,28 +22,68 @@ from tempest.lib import exceptions class ObjectStorageService(Service): - def set_extensions(self, object_store_discovery=False): - if not object_store_discovery: + def set_extensions(self): + if 'v3' not in self.service_url: # it's not a v3 url + try: + body = self.do_get(self.service_url, top_level=True, + top_level_path="info") + body = json.loads(body) + # Remove Swift general information from extensions list + body.pop('swift') + self.extensions = body.keys() + except Exception: + self.extensions = [] + else: self.extensions = [] - elif 'v3' not in self.service_url: # it's not a v3 url - body = self.do_get(self.service_url, top_level=True, - top_level_path="info") - body = json.loads(body) - # Remove Swift general information from extensions list - body.pop('swift') - self.extensions = body.keys() def list_create_roles(self, conf, client): try: roles = client.list_roles()['roles'] except exceptions.Forbidden: LOG.info("Roles can't be listed - the user needs permissions.") + # If is not admin, we set the operator_role to Member + # otherwise we set to admin + conf.set('object-storage', 'operator_role', 'Member') return - for section_key in ["operator_role", "reseller_admin"]: + for section_key in ["operator_role", "reseller_admin_role"]: key_value = conf.get_defaulted("object-storage", section_key) if key_value not in [r['name'] for r in roles]: LOG.info("Creating %s role", key_value) - client.create_role(name=key_value) + try: + client.create_role(name=key_value) + except exceptions.Conflict: + LOG.info("Role %s already exists", key_value) + conf.set('object-storage', 'operator_role', 'admin') - conf.set("object-storage", section_key, key_value) + def check_service_status(self, conf): + """Use healthcheck api to check the service status + + :type conf: TempestConf object + """ + # Check for swift discoverability if it is False + # check_service_status returns False + # Else above is True, then we can check for healthcheck + # API then we can find the service_status + try: + if not conf.get_bool_value( + conf.get( + 'object-storage-feature-enabled', + 'discoverability')): + return False + except ConfigParser.NoSectionError: + # Turning http://.../v1/foobar into http://.../ + self.client.accounts.skip_path() + resp, _ = self.client.accounts.get("healthcheck", {}) + return resp['status'] == '200' + except Exception: + return False + + def set_default_tempest_options(self, conf): + """Set default values for swift + + """ + swift_status = self.check_service_status(conf) + # Set roles based on service status + if swift_status: + self.list_create_roles(conf, self.client.roles) diff --git a/config_tempest/services/services.py b/config_tempest/services/services.py index 808badcd..d7816683 100644 --- a/config_tempest/services/services.py +++ b/config_tempest/services/services.py @@ -42,9 +42,6 @@ class Services(object): self._clients = clients self._conf = conf self._creds = creds - swift_discover = conf.get_defaulted('object-storage-feature-enabled', - 'discoverability') - self._object_store_discovery = conf.get_bool_value(swift_discover) self._ssl_validation = creds.disable_ssl_certificate_validation self._region = clients.identity_region self._services = [] @@ -63,10 +60,7 @@ class Services(object): service = service_class(name, url, token, self._ssl_validation, self._clients.get_service_client(name)) # discover extensions of the service - if name == 'object-store': - service.set_extensions(self._object_store_discovery) - else: - service.set_extensions() + service.set_extensions() # discover versions of the service service.set_versions() @@ -204,15 +198,6 @@ class Services(object): self._clients.volume_client, self.is_service("volumev3")) - # query the config for swift availability and get the current value - # in case, it was overridden in CLI - swift_default = self._conf.get_bool_value( - self._conf.get_defaulted('service_available', 'swift') - ) - if self.is_service('object-store') and swift_default: - object_storage = self.get_service('object-store') - object_storage.list_create_roles(self._conf, self._clients.roles) - ceilometer.check_ceilometer_service(self._conf, self._clients.service_client) @@ -282,10 +267,4 @@ class Services(object): service_object = self.get_service(service) if service_object is not None: extensions = ','.join(service_object.get_extensions()) - - if service == 'object-store': - # tempest.conf is inconsistent and uses 'object-store' for - # the catalog name but 'object-storage-feature-enabled' - service = 'object-storage' - self._conf.set(service + postfix, ext_key, extensions) diff --git a/config_tempest/tests/base.py b/config_tempest/tests/base.py index 35f8fa69..ee6ee286 100644 --- a/config_tempest/tests/base.py +++ b/config_tempest/tests/base.py @@ -211,6 +211,18 @@ class BaseServiceTest(base.BaseTestCase): ] } ) + FAKE_ACCOUNTS = ( + { + 'status': '200', + u'content-length': '2', + 'content-location': 'http://192.168.122.120:8080/healthcheck', + u'connection': 'close', + u'x-trans-id': 'txec03483c96cd4958a5c6b-005b17c346', + u'date': 'Wed, 06 Jun 2018 11:19:34 GMT', + u'content-type': 'text/plain', + u'x-openstack-request-id': 'txec03483c96cd4958a5c6b-005b17c346' + }, + 'OK') class FakeRequestResponse(object): URL = 'https://docs.openstack.org/api/openstack-identity/3/ext/' diff --git a/config_tempest/tests/services/test_object_storage.py b/config_tempest/tests/services/test_object_storage.py index 5f5e6e99..a411945f 100644 --- a/config_tempest/tests/services/test_object_storage.py +++ b/config_tempest/tests/services/test_object_storage.py @@ -28,14 +28,18 @@ class TestObjectStorageService(BaseServiceTest): self.FAKE_URL, self.FAKE_TOKEN, disable_ssl_validation=False) + self.Service.conf = tempest_conf.TempestConf() def test_set_get_extensions(self): expected_resp = ['formpost', 'ratelimit', 'methods', 'account_quotas'] self._fake_service_do_get_method(self.FAKE_STORAGE_EXTENSIONS) - self.Service.set_extensions(object_store_discovery=True) + self.Service.set_extensions() self.assertItemsEqual(self.Service.extensions, expected_resp) self.assertItemsEqual(self.Service.get_extensions(), expected_resp) + + def test_set_get_extensions_empty(self): + self.Service.service_url = self.FAKE_URL + 'v3' self.Service.set_extensions() self.assertItemsEqual(self.Service.extensions, []) self.assertItemsEqual(self.Service.get_extensions(), []) @@ -49,9 +53,17 @@ class TestObjectStorageService(BaseServiceTest): client.list_roles = return_mock client.create_role = mock.Mock() self.Service.list_create_roles(conf, client) - self.assertEqual(conf.get('object-storage', 'reseller_admin'), + self.assertEqual(conf.get('object-storage', 'reseller_admin_role'), 'ResellerAdmin') - # Member role is inherited from tempest.config self.assertEqual(conf.get('object-storage', 'operator_role'), - 'Member') + 'admin') self.assertTrue(client.create_role.called) + + def test_check_service_status(self): + self.Service.client = mock.Mock() + self.Service.client.accounts = mock.Mock() + return_mock = mock.Mock(return_value=self.FAKE_ACCOUNTS) + self.Service.client.accounts.skip_check = mock.Mock() + self.Service.client.accounts.get = return_mock + self.Service.check_service_status(self.Service.conf) + self.assertTrue(self.Service.check_service_status) diff --git a/etc/default-overrides.conf b/etc/default-overrides.conf index 3be569fb..942975d8 100644 --- a/etc/default-overrides.conf +++ b/etc/default-overrides.conf @@ -62,7 +62,7 @@ disable_ssl_certificate_validation=true [object-storage] # default-overrides.conf file will be removed soon, this value will be # moved to load_basic_defaults method in config_tempest/main.py -reseller_admin = ResellerAdmin +reseller_admin_role = ResellerAdmin [data-processing] diff --git a/releasenotes/notes/use-healthcheck-api-for-swift-e84cbb999be4ec3d.yaml b/releasenotes/notes/use-healthcheck-api-for-swift-e84cbb999be4ec3d.yaml new file mode 100644 index 00000000..4e425277 --- /dev/null +++ b/releasenotes/notes/use-healthcheck-api-for-swift-e84cbb999be4ec3d.yaml @@ -0,0 +1,8 @@ +--- +prelude: > + Use healthcheck api for determine swift service availability +features: + - | + Discover swift service only when healthcheck api is working otherwise + set it to false. + It also removes hardcoded values from swift. diff --git a/roles/generate-accounts-file/defaults/main.yaml b/roles/generate-accounts-file/defaults/main.yaml index c9996bb9..5d8a1761 100644 --- a/roles/generate-accounts-file/defaults/main.yaml +++ b/roles/generate-accounts-file/defaults/main.yaml @@ -1,3 +1,7 @@ -tempest_concurrency: 2 +# Here, we set tempest_concurrency to 3 because with concurrency 2 generate +# only 10 accounts, and sometimes the tests fail because the account is being +# used by another test, so it's a good idea to have tempest_concurrency always +# bigger then tempest test concurrency (in our jobs is set to 2) +tempest_concurrency: 3 virtualenvs: tempest: ~/.virtualenvs/.tempest diff --git a/roles/generate-tempestconf-file-cloud/tasks/main.yaml b/roles/generate-tempestconf-file-cloud/tasks/main.yaml index e4bf4e97..8e57ea4d 100644 --- a/roles/generate-tempestconf-file-cloud/tasks/main.yaml +++ b/roles/generate-tempestconf-file-cloud/tasks/main.yaml @@ -39,8 +39,7 @@ --non-admin \ {% endif %} --os-cloud {{ cloud_user }} \ - auth.tempest_roles Member \ - service_available.swift False + auth.tempest_roles Member args: chdir: "{{ tempestconf_src_relative_path }}" executable: /bin/bash diff --git a/roles/generate-tempestconf-file/tasks/generate-tempestconf.sh.j2 b/roles/generate-tempestconf-file/tasks/generate-tempestconf.sh.j2 index 68ada88c..ab456b98 100644 --- a/roles/generate-tempestconf-file/tasks/generate-tempestconf.sh.j2 +++ b/roles/generate-tempestconf-file/tasks/generate-tempestconf.sh.j2 @@ -17,5 +17,4 @@ discover-tempest-config \ {% endif %} identity.uri $OS_AUTH_URL \ auth.admin_password $OS_PASSWORD \ -service_available.swift False \ {{ aditional_tempestconf_params }}