openinfra/python-tempestconf
Code Issues Proposed changes

Use healthcheck api to determine swift service

Browse Source 
* Added check_service_status to determine the service
  availability and later on it will be used for all services.

* Removed hardcoded values for swift services
* As swift was disabled earlier, SwiftOperator was not getting used
  and it is not present in the CI Job, let's use member role for
  the same.
* Set operator_role default to admin if admin credential is available
  otherwise, set operator_role to ResellerAdmin
* Do not fail if a conflict exist in roles names

Story: 2001253
Task: 5783

Co-Authored-By: Arx Cruz <arxcruz@redhat.com>
Depends-On: https://review.openstack.org/#/c/576472/
Closes-Bug: 1776729
Change-Id: Ie1e9d8e98fde460f9270c2799f971ea017d10d84
This commit is contained in:
Chandan Kumar 2018-06-14 22:02:55 +05:30 committed by Arx Cruz
parent 6935a898bb
commit f5bfc21351
10 changed files with 105 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
config_tempest/clients.py
View File

@ -30,6 +30,7 @@ from tempest.lib.services.identity.v3 import services_client as s_client
from tempest.lib.services.identity.v3 import users_client as users_v3_client
from tempest.lib.services.image.v2 import images_client
from tempest.lib.services.network import networks_client
from tempest.lib.services.object_storage import account_client
try:
# Since Rocky, volume.v3.services_client is the default
from tempest.lib.services.volume.v3 import services_client
@ -119,6 +120,12 @@ class ClientManager(object):
self.identity_region,
**default_params)
self.accounts = account_client.AccountClient(
self.auth_provider,
conf.get_defaulted('object-storage', 'catalog_type'),
self.identity_region,
**default_params)
self.set_users_client(
auth=self.auth_provider,
identity_version=creds.identity_version,
@ -227,7 +234,7 @@ class ClientManager(object):
# and so on.
if service_name == "image":
return self.images
elif service_name == "network":
elif service_name in ["network", "object-store"]:
# return whole ClientManager object because NetworkService
# currently needs to have an access to get_neutron/nova_client
# methods which are chosen according to neutron presence

65
config_tempest/services/object_storage.py
View File

@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import json
from base import Service
@ -21,28 +22,68 @@ from tempest.lib import exceptions
class ObjectStorageService(Service):
def set_extensions(self, object_store_discovery=False):
if not object_store_discovery:
def set_extensions(self):
if 'v3' not in self.service_url: # it's not a v3 url
try:
body = self.do_get(self.service_url, top_level=True,
top_level_path="info")
body = json.loads(body)
# Remove Swift general information from extensions list
body.pop('swift')
self.extensions = body.keys()
except Exception:
self.extensions = []
else:
self.extensions = []
elif 'v3' not in self.service_url: # it's not a v3 url
body = self.do_get(self.service_url, top_level=True,
top_level_path="info")
body = json.loads(body)
# Remove Swift general information from extensions list
body.pop('swift')
self.extensions = body.keys()
def list_create_roles(self, conf, client):
try:
roles = client.list_roles()['roles']
except exceptions.Forbidden:
LOG.info("Roles can't be listed - the user needs permissions.")
# If is not admin, we set the operator_role to Member
# otherwise we set to admin
conf.set('object-storage', 'operator_role', 'Member')
return
for section_key in ["operator_role", "reseller_admin"]:
for section_key in ["operator_role", "reseller_admin_role"]:
key_value = conf.get_defaulted("object-storage", section_key)
if key_value not in [r['name'] for r in roles]:
LOG.info("Creating %s role", key_value)
client.create_role(name=key_value)
try:
client.create_role(name=key_value)
except exceptions.Conflict:
LOG.info("Role %s already exists", key_value)
conf.set('object-storage', 'operator_role', 'admin')
conf.set("object-storage", section_key, key_value)
def check_service_status(self, conf):
"""Use healthcheck api to check the service status
:type conf: TempestConf object
"""
# Check for swift discoverability if it is False
# check_service_status returns False
# Else above is True, then we can check for healthcheck
# API then we can find the service_status
try:
if not conf.get_bool_value(
conf.get(
'object-storage-feature-enabled',
'discoverability')):
return False
except ConfigParser.NoSectionError:
# Turning http://.../v1/foobar into http://.../
self.client.accounts.skip_path()
resp, _ = self.client.accounts.get("healthcheck", {})
return resp['status'] == '200'
except Exception:
return False
def set_default_tempest_options(self, conf):
"""Set default values for swift
"""
swift_status = self.check_service_status(conf)
# Set roles based on service status
if swift_status:
self.list_create_roles(conf, self.client.roles)

23
config_tempest/services/services.py
View File

@ -40,9 +40,6 @@ class Services(object):
self._clients = clients
self._conf = conf
self._creds = creds
swift_discover = conf.get_defaulted('object-storage-feature-enabled',
'discoverability')
self._object_store_discovery = conf.get_bool_value(swift_discover)
self._ssl_validation = creds.disable_ssl_certificate_validation
self._region = clients.identity_region
self._services = []
@ -61,10 +58,7 @@ class Services(object):
service = service_class(name, url, token, self._ssl_validation,
self._clients.get_service_client(name))
# discover extensions of the service
if name == 'object-store':
service.set_extensions(self._object_store_discovery)
else:
service.set_extensions()
service.set_extensions()
# discover versions of the service
service.set_versions()
@ -202,15 +196,6 @@ class Services(object):
self._clients.volume_client,
self.is_service("volumev3"))
# query the config for swift availability and get the current value
# in case, it was overridden in CLI
swift_default = self._conf.get_bool_value(
self._conf.get_defaulted('service_available', 'swift')
)
if self.is_service('object-store') and swift_default:
object_storage = self.get_service('object-store')
object_storage.list_create_roles(self._conf, self._clients.roles)
ceilometer.check_ceilometer_service(self._conf,
self._clients.service_client)
@ -282,10 +267,4 @@ class Services(object):
service_object = self.get_service(service)
if service_object is not None:
extensions = ','.join(service_object.get_extensions())
if service == 'object-store':
# tempest.conf is inconsistent and uses 'object-store' for
# the catalog name but 'object-storage-feature-enabled'
service = 'object-storage'
self._conf.set(service + postfix, ext_key, extensions)

12
config_tempest/tests/base.py
View File

@ -211,6 +211,18 @@ class BaseServiceTest(base.BaseTestCase):
]
}
)
FAKE_ACCOUNTS = (
{
'status': '200',
u'content-length': '2',
'content-location': 'http://192.168.122.120:8080/healthcheck',
u'connection': 'close',
u'x-trans-id': 'txec03483c96cd4958a5c6b-005b17c346',
u'date': 'Wed, 06 Jun 2018 11:19:34 GMT',
u'content-type': 'text/plain',
u'x-openstack-request-id': 'txec03483c96cd4958a5c6b-005b17c346'
},
'OK')
class FakeRequestResponse(object):
URL = 'https://docs.openstack.org/api/openstack-identity/3/ext/'

20
config_tempest/tests/services/test_object_storage.py
View File

@ -28,14 +28,18 @@ class TestObjectStorageService(BaseServiceTest):
self.FAKE_URL,
self.FAKE_TOKEN,
disable_ssl_validation=False)
self.Service.conf = tempest_conf.TempestConf()
def test_set_get_extensions(self):
expected_resp = ['formpost', 'ratelimit',
'methods', 'account_quotas']
self._fake_service_do_get_method(self.FAKE_STORAGE_EXTENSIONS)
self.Service.set_extensions(object_store_discovery=True)
self.Service.set_extensions()
self.assertItemsEqual(self.Service.extensions, expected_resp)
self.assertItemsEqual(self.Service.get_extensions(), expected_resp)
def test_set_get_extensions_empty(self):
self.Service.service_url = self.FAKE_URL + 'v3'
self.Service.set_extensions()
self.assertItemsEqual(self.Service.extensions, [])
self.assertItemsEqual(self.Service.get_extensions(), [])
@ -49,9 +53,17 @@ class TestObjectStorageService(BaseServiceTest):
client.list_roles = return_mock
client.create_role = mock.Mock()
self.Service.list_create_roles(conf, client)
self.assertEqual(conf.get('object-storage', 'reseller_admin'),
self.assertEqual(conf.get('object-storage', 'reseller_admin_role'),
'ResellerAdmin')
# Member role is inherited from tempest.config
self.assertEqual(conf.get('object-storage', 'operator_role'),
'Member')
'admin')
self.assertTrue(client.create_role.called)
def test_check_service_status(self):
self.Service.client = mock.Mock()
self.Service.client.accounts = mock.Mock()
return_mock = mock.Mock(return_value=self.FAKE_ACCOUNTS)
self.Service.client.accounts.skip_check = mock.Mock()
self.Service.client.accounts.get = return_mock
self.Service.check_service_status(self.Service.conf)
self.assertTrue(self.Service.check_service_status)

2
etc/default-overrides.conf
View File

@ -62,7 +62,7 @@ disable_ssl_certificate_validation=true
[object-storage]
# default-overrides.conf file will be removed soon, this value will be
# moved to load_basic_defaults method in config_tempest/main.py
reseller_admin = ResellerAdmin
reseller_admin_role = ResellerAdmin
[data-processing]

8
releasenotes/notes/use-healthcheck-api-for-swift-e84cbb999be4ec3d.yaml Normal file
View File

@ -0,0 +1,8 @@
---
prelude: >
Use healthcheck api for determine swift service availability
features:
- |
Discover swift service only when healthcheck api is working otherwise
set it to false.
It also removes hardcoded values from swift.

6
roles/generate-accounts-file/defaults/main.yaml
View File

@ -1,3 +1,7 @@
tempest_concurrency: 2
# Here, we set tempest_concurrency to 3 because with concurrency 2 generate
# only 10 accounts, and sometimes the tests fail because the account is being
# used by another test, so it's a good idea to have tempest_concurrency always
# bigger then tempest test concurrency (in our jobs is set to 2)
tempest_concurrency: 3
virtualenvs:
tempest: ~/.virtualenvs/.tempest

3
roles/generate-tempestconf-file-cloud/tasks/main.yaml
View File

@ -39,8 +39,7 @@
--non-admin \
{% endif %}
--os-cloud {{ cloud_user }} \
auth.tempest_roles Member \
service_available.swift False
auth.tempest_roles Member
args:
chdir: "{{ tempestconf_src_relative_path }}"
executable: /bin/bash

1
roles/generate-tempestconf-file/tasks/generate-tempestconf.sh.j2
View File

@ -17,5 +17,4 @@ discover-tempest-config \
{% endif %}
identity.uri $OS_AUTH_URL \
auth.admin_password $OS_PASSWORD \
service_available.swift False \
{{ aditional_tempestconf_params }}