starlingx
/
distcloud
Code Issues Proposed changes

Restructure subcloud audit to audit subclouds in parallel 

Restructuring the dcmanager subcloud audit to audit subclouds
in parallel by creating a greenthread for each subcloud as it
is audited. This reduces the impact on the audit time when
a particular subcloud audit takes a longer amount of time
(e.g. when the subcloud is powered down).

Also updated the EndpointCache class to use the configured
http_connect_timeout (10s) when creating a keystone session.
This reduces the amount of time it takes to timeout a
failed connection to 10s from several minutes.

Finally, added a significant number of unit tests for the
dcmanager subcloud audit.

Change-Id: Ic56470d2f5232c1481730fe4782d27b34089395b
Closes-Bug: 185494
Signed-off-by: Bart Wensley <barton.wensley@windriver.com>
This commit is contained in:
Bart Wensley 2020-01-20 14:11:20 -06:00
parent ab11f85898
commit 86d536ac52
9 changed files with 643 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
distributedcloud/dcmanager/cmd/manager.py
View File

@ -56,6 +56,10 @@ def main():
consts.TOPIC_DC_MANAGER)
launcher = service.launch(cfg.CONF,
srv, workers=cfg.CONF.workers)
LOG.info("Configuration:")
cfg.CONF.log_opt_values(LOG, logging.INFO)
# the following periodic tasks are intended serve as HA checking
# srv.create_periodic_tasks()
launcher.wait()

4
distributedcloud/dcmanager/manager/patch_audit_manager.py
View File

@ -144,8 +144,8 @@ class PatchAuditManager(manager.Manager):
try:
sc_ks_client = KeystoneClient(subcloud.name)
except (keystone_exceptions.EndpointNotFound, IndexError) as e:
LOG.warn("Identity endpoint for online subcloud % not found. %"
% (subcloud.name, e))
LOG.warn("Identity endpoint for online subcloud %s not found."
" %s" % (subcloud.name, e))
continue
try:

4
distributedcloud/dcmanager/manager/scheduler.py
View File

@ -32,10 +32,10 @@ wallclock = time.time
class ThreadGroupManager(object):
"""Thread group manager."""
def __init__(self):
def __init__(self, *args, **kwargs):
super(ThreadGroupManager, self).__init__()
self.threads = {}
self.group = threadgroup.ThreadGroup()
self.group = threadgroup.ThreadGroup(*args, **kwargs)
def start(self, func, *args, **kwargs):
"""Run the given method in a sub-thread."""

4
distributedcloud/dcmanager/manager/service.py
View File

@ -117,9 +117,9 @@ class DCManagerService(service.Service):
if self.periodic_enable:
LOG.info("Adding periodic tasks for the manager to perform")
self.TG.add_timer(cfg.CONF.scheduler.subcloud_audit_interval,
self.subcloud_audit, None)
self.subcloud_audit, initial_delay=10)
self.TG.add_timer(cfg.CONF.scheduler.patch_audit_interval,
self.patch_audit, None)
self.patch_audit, initial_delay=60)
def subcloud_audit(self):
# Audit availability of all subclouds.

526
distributedcloud/dcmanager/manager/subcloud_audit_manager.py
View File

@ -13,15 +13,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017-2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement.
#
from keystoneauth1 import exceptions as keystone_exceptions
from oslo_log import log as logging
from fm_api import constants as fm_const
from fm_api import fm_api
from sysinv.common import constants as sysinv_constants
from dcorch.common import consts as dcorch_consts
from dcorch.drivers.openstack.keystone_v3 import KeystoneClient
from dcorch.rpc import client as dcorch_rpc_client
@ -32,13 +37,8 @@ from dcmanager.common import exceptions
from dcmanager.common.i18n import _
from dcmanager.common import manager
from dcmanager.db import api as db_api
from dcmanager.drivers.openstack.sysinv_v1 import SysinvClient
from keystoneauth1 import exceptions as keystone_exceptions
from fm_api import constants as fm_const
from fm_api import fm_api
from sysinv.common import constants as sysinv_constants
from dcmanager.manager import scheduler
LOG = logging.getLogger(__name__)
@ -55,6 +55,11 @@ class SubcloudAuditManager(manager.Manager):
self.dcorch_rpc_client = dcorch_rpc_client.EngineClient()
self.fm_api = fm_api.FaultAPIs()
self.subcloud_manager = kwargs['subcloud_manager']
# Keeps track of greenthreads we create to do work.
self.thread_group_manager = scheduler.ThreadGroupManager(
thread_pool_size=100)
# Track workers created for each subcloud.
self.subcloud_workers = dict()
def periodic_subcloud_audit(self):
"""Audit availability of subclouds."""
@ -68,253 +73,302 @@ class SubcloudAuditManager(manager.Manager):
def _periodic_subcloud_audit_loop(self):
"""Audit availability of subclouds loop."""
# We will be running in our own green thread here.
LOG.info('Triggered subcloud audit.')
# Determine whether OpenStack is installed in central cloud
ks_client = KeystoneClient()
sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME,
ks_client.session)
# This could be optimized in the future by attempting to get just the
# one application. However, sysinv currently treats this as a failure
# if the application is not installed and generates warning logs, so it
# would require changes to handle this gracefully.
apps = sysinv_client.get_applications()
openstack_installed = False
for app in apps:
if app.name == sysinv_constants.HELM_APP_OPENSTACK and app.active:
openstack_installed = True
break
for subcloud in db_api.subcloud_get_all(self.context):
# Create a new greenthread for each subcloud to allow the audits
# to be done in parallel. If there are not enough greenthreads
# in the pool, this will block until one becomes available.
self.subcloud_workers[subcloud.name] = \
self.thread_group_manager.start(self._audit_subcloud,
subcloud.name,
openstack_installed)
# Wait for all greenthreads to complete
LOG.info('Waiting for subcloud audits to complete.')
for thread in self.subcloud_workers.values():
thread.wait()
# Clear the list of workers before next audit
self.subcloud_workers = dict()
LOG.info('All subcloud audits have completed.')
def _audit_subcloud(self, subcloud_name, audit_openstack):
"""Audit a single subcloud."""
# Retrieve the subcloud
try:
subcloud = db_api.subcloud_get_by_name(self.context, subcloud_name)
except exceptions.SubcloudNotFound:
# Possibility subcloud could have been deleted since the list of
# subclouds to audit was created.
LOG.info('Ignoring SubcloudNotFound when auditing subcloud %s' %
subcloud_name)
return
# For each subcloud, if at least one service is active in
# each service of servicegroup-list then declare the subcloud online.
for subcloud in db_api.subcloud_get_all(self.context):
subcloud_name = subcloud.name
subcloud_id = subcloud.id
management_state = subcloud.management_state
avail_status_current = subcloud.availability_status
audit_fail_count = subcloud.audit_fail_count
openstack_installed = subcloud.openstack_installed
subcloud_id = subcloud.id
avail_status_current = subcloud.availability_status
audit_fail_count = subcloud.audit_fail_count
# Set defaults to None and disabled so we will still set disabled
# status if we encounter an error.
# Set defaults to None and disabled so we will still set disabled
# status if we encounter an error.
sysinv_client = None
svc_groups = None
avail_to_set = consts.AVAILABILITY_OFFLINE
sysinv_client = None
svc_groups = None
avail_to_set = consts.AVAILABILITY_OFFLINE
try:
ks_client = KeystoneClient(subcloud_name)
sysinv_client = SysinvClient(subcloud_name,
ks_client.session)
except (keystone_exceptions.EndpointNotFound,
keystone_exceptions.ConnectFailure, IndexError) as e:
if avail_status_current == consts.AVAILABILITY_OFFLINE:
LOG.info("Identity or Platform endpoint for %s not "
"found, ignoring for offline "
"subcloud." % subcloud_name)
continue
else:
LOG.error("Identity or Platform endpoint for online "
"subcloud: %s not found." % subcloud_name)
except Exception as e:
LOG.exception(e)
if sysinv_client:
# get a list of service groups in the subcloud
try:
svc_groups = sysinv_client.get_service_groups()
except Exception as e:
svc_groups = None
LOG.warn('Cannot retrieve service groups for '
'subcloud:%s, %s' % (subcloud_name, e))
if svc_groups:
active_sgs = []
inactive_sgs = []
# Build 2 lists, 1 of active service groups,
# one with non-active.
for sg in svc_groups:
if sg.state != consts.SERVICE_GROUP_STATUS_ACTIVE:
inactive_sgs.append(sg.service_group_name)
else:
active_sgs.append(sg.service_group_name)
# Create a list of service groups that are only present
# in non-active list
inactive_only = [sg for sg in inactive_sgs if
sg not in active_sgs]
# An empty inactive only list and a non-empty active list
# means we're good to go.
if not inactive_only and active_sgs:
avail_to_set = \
consts.AVAILABILITY_ONLINE
else:
LOG.info("Subcloud:%s has non-active "
"service groups: %s" %
(subcloud_name, inactive_only))
if avail_to_set == consts.AVAILABILITY_OFFLINE:
if audit_fail_count < consts.AVAIL_FAIL_COUNT_MAX:
audit_fail_count = audit_fail_count + 1
if (avail_status_current == consts.AVAILABILITY_ONLINE) and \
(audit_fail_count < consts.AVAIL_FAIL_COUNT_TO_ALARM):
# Do not set offline until we have failed audit
# the requisite number of times
avail_to_set = consts.AVAILABILITY_ONLINE
try:
ks_client = KeystoneClient(subcloud_name)
sysinv_client = SysinvClient(subcloud_name,
ks_client.session)
except (keystone_exceptions.EndpointNotFound,
keystone_exceptions.ConnectFailure,
keystone_exceptions.ConnectTimeout,
IndexError):
if avail_status_current == consts.AVAILABILITY_OFFLINE:
LOG.info("Identity or Platform endpoint for %s not "
"found, ignoring for offline "
"subcloud." % subcloud_name)
return
else:
# In the case of a one off blip, we may need to set the
# fail count back to 0
# The subcloud will be marked as offline below.
LOG.error("Identity or Platform endpoint for online "
"subcloud: %s not found." % subcloud_name)
except Exception as e:
LOG.exception(e)
if sysinv_client:
# get a list of service groups in the subcloud
try:
svc_groups = sysinv_client.get_service_groups()
except Exception as e:
svc_groups = None
LOG.warn('Cannot retrieve service groups for '
'subcloud: %s, %s' % (subcloud_name, e))
if svc_groups:
active_sgs = []
inactive_sgs = []
# Build 2 lists, 1 of active service groups,
# one with non-active.
for sg in svc_groups:
if sg.state != consts.SERVICE_GROUP_STATUS_ACTIVE:
inactive_sgs.append(sg.service_group_name)
else:
active_sgs.append(sg.service_group_name)
# Create a list of service groups that are only present
# in non-active list
inactive_only = [sg for sg in inactive_sgs if
sg not in active_sgs]
# An empty inactive only list and a non-empty active list
# means we're good to go.
if not inactive_only and active_sgs:
avail_to_set = \
consts.AVAILABILITY_ONLINE
else:
LOG.info("Subcloud:%s has non-active "
"service groups: %s" %
(subcloud_name, inactive_only))
if avail_to_set == consts.AVAILABILITY_OFFLINE:
if audit_fail_count < consts.AVAIL_FAIL_COUNT_MAX:
audit_fail_count = audit_fail_count + 1
if (avail_status_current == consts.AVAILABILITY_ONLINE) and \
(audit_fail_count < consts.AVAIL_FAIL_COUNT_TO_ALARM):
# Do not set offline until we have failed audit
# the requisite number of times
avail_to_set = consts.AVAILABILITY_ONLINE
else:
# In the case of a one off blip, we may need to set the
# fail count back to 0
audit_fail_count = 0
if avail_to_set != avail_status_current:
if avail_to_set == consts.AVAILABILITY_ONLINE:
audit_fail_count = 0
if avail_to_set != avail_status_current:
LOG.info('Setting new availability status: %s '
'on subcloud: %s' %
(avail_to_set, subcloud_name))
if avail_to_set == consts.AVAILABILITY_ONLINE:
audit_fail_count = 0
LOG.info('Setting new availability status: %s '
'on subcloud: %s' %
(avail_to_set, subcloud_name))
entity_instance_id = "subcloud=%s" % subcloud_name
fault = self.fm_api.get_fault(
fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
entity_instance_id)
if fault and (avail_to_set == consts.AVAILABILITY_ONLINE):
try:
self.fm_api.clear_fault(
fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
entity_instance_id)
except Exception as e:
LOG.exception(e)
elif not fault and \
(avail_to_set == consts.AVAILABILITY_OFFLINE):
try:
fault = fm_api.Fault(
alarm_id=fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
alarm_state=fm_const.FM_ALARM_STATE_SET,
entity_type_id=fm_const.FM_ENTITY_TYPE_SUBCLOUD,
entity_instance_id=entity_instance_id,
severity=fm_const.FM_ALARM_SEVERITY_CRITICAL,
reason_text=('%s is offline' % subcloud_name),
alarm_type=fm_const.FM_ALARM_TYPE_0,
probable_cause=fm_const.ALARM_PROBABLE_CAUSE_29,
proposed_repair_action="Wait for subcloud to "
"become online; if "
"problem persists contact "
"next level of support.",
service_affecting=True)
self.fm_api.set_fault(fault)
except Exception as e:
LOG.exception(e)
entity_instance_id = "subcloud=%s" % subcloud_name
fault = self.fm_api.get_fault(
fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
entity_instance_id)
if fault and (avail_to_set == consts.AVAILABILITY_ONLINE):
try:
db_api.subcloud_update(self.context, subcloud_id,
management_state=None,
availability_status=avail_to_set,
software_version=None,
description=None, location=None,
audit_fail_count=audit_fail_count)
self.fm_api.clear_fault(
fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
entity_instance_id)
except Exception as e:
LOG.exception(e)
elif not fault and \
(avail_to_set == consts.AVAILABILITY_OFFLINE):
try:
fault = fm_api.Fault(
alarm_id=fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
alarm_state=fm_const.FM_ALARM_STATE_SET,
entity_type_id=fm_const.FM_ENTITY_TYPE_SUBCLOUD,
entity_instance_id=entity_instance_id,
severity=fm_const.FM_ALARM_SEVERITY_CRITICAL,
reason_text=('%s is offline' % subcloud_name),
alarm_type=fm_const.FM_ALARM_TYPE_0,
probable_cause=fm_const.ALARM_PROBABLE_CAUSE_29,
proposed_repair_action="Wait for subcloud to "
"become online; if "
"problem persists contact "
"next level of support.",
service_affecting=True)
self.fm_api.set_fault(fault)
except Exception as e:
LOG.exception(e)
try:
updated_subcloud = db_api.subcloud_update(
self.context,
subcloud_id,
management_state=None,
availability_status=avail_to_set,
software_version=None,
description=None, location=None,
audit_fail_count=audit_fail_count)
except exceptions.SubcloudNotFound:
# slim possibility subcloud could have been deleted since
# we found it in db, ignore this benign error.
LOG.info('Ignoring SubcloudNotFound when attempting state'
' update: %s' % subcloud_name)
return
try:
self.dcorch_rpc_client.\
update_subcloud_states(self.context,
subcloud_name,
updated_subcloud.management_state,
avail_to_set)
LOG.info('Notifying dcorch, subcloud:%s management: %s, '
'availability:%s' %
(subcloud_name,
updated_subcloud.management_state,
avail_to_set))
except Exception as e:
LOG.exception(e)
LOG.warn('Problem informing dcorch of subcloud '
'state change, subcloud: %s' % subcloud_name)
if avail_to_set == consts.AVAILABILITY_OFFLINE:
# Subcloud is going offline, set all endpoint statuses to
# unknown.
try:
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud_name,
endpoint_type=None,
sync_status=consts.SYNC_STATUS_UNKNOWN)
except exceptions.SubcloudNotFound:
# slim possibility subcloud could have been deleted since
# we found it in db, ignore this benign error.
LOG.info('Ignoring SubcloudNotFound when attempting state'
' update: %s' % subcloud_name)
continue
LOG.info('Ignoring SubcloudNotFound when attempting '
'sync_status update: %s' % subcloud_name)
return
elif audit_fail_count != subcloud.audit_fail_count:
try:
db_api.subcloud_update(self.context, subcloud_id,
management_state=None,
availability_status=None,
software_version=None,
description=None, location=None,
audit_fail_count=audit_fail_count)
except exceptions.SubcloudNotFound:
# slim possibility subcloud could have been deleted since
# we found it in db, ignore this benign error.
LOG.info('Ignoring SubcloudNotFound when attempting '
'audit_fail_count update: %s' % subcloud_name)
return
if audit_openstack and sysinv_client:
# get a list of installed apps in the subcloud
try:
apps = sysinv_client.get_applications()
except Exception as e:
LOG.warn('Cannot retrieve installed apps for '
'subcloud:%s, %s' % (subcloud_name, e))
return
openstack_installed = subcloud.openstack_installed
openstack_installed_current = False
for app in apps:
if app.name == sysinv_constants.HELM_APP_OPENSTACK\
and app.active:
# audit find openstack app is installed and active in
# the subcloud
openstack_installed_current = True
break
dcm_update_func = None
dco_update_func = None
if openstack_installed_current and not openstack_installed:
dcm_update_func = db_api.subcloud_status_create
dco_update_func = self.dcorch_rpc_client.\
add_subcloud_sync_endpoint_type
elif not openstack_installed_current and openstack_installed:
dcm_update_func = db_api.subcloud_status_delete
dco_update_func = self.dcorch_rpc_client.\
remove_subcloud_sync_endpoint_type
if dcm_update_func and dco_update_func:
endpoint_type_list = dcorch_consts.ENDPOINT_TYPES_LIST_OS
try:
self.dcorch_rpc_client.\
update_subcloud_states(self.context,
subcloud_name,
management_state,
avail_to_set)
LOG.info('Notifying dcorch, subcloud:%s management: %s, '
'availability:%s' % (subcloud_name,
management_state,
avail_to_set))
# Notify dcorch to add/remove sync endpoint type list
dco_update_func(self.context, subcloud_name,
endpoint_type_list)
LOG.info('Notifying dcorch, subcloud: %s new sync'
' endpoint: %s' % (subcloud_name,
endpoint_type_list))
# Update subcloud status table by adding/removing
# openstack sync endpoint types.
for endpoint_type in endpoint_type_list:
dcm_update_func(self.context, subcloud_id,
endpoint_type)
# Update openstack_installed of subcloud table
db_api.subcloud_update(
self.context, subcloud_id,
openstack_installed=openstack_installed_current)
except exceptions.SubcloudNotFound:
LOG.info('Ignoring SubcloudNotFound when attempting'
' openstack_installed update: %s'
% subcloud_name)
except Exception as e:
LOG.exception(e)
LOG.warn('Problem informing dcorch of subcloud '
'state change, subcloud: %s' % subcloud_name)
if avail_to_set == consts.AVAILABILITY_OFFLINE:
# Subcloud is going offline, set all endpoint statuses to
# unknown.
try:
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud_name,
endpoint_type=None,
sync_status=consts.SYNC_STATUS_UNKNOWN)
except exceptions.SubcloudNotFound:
LOG.info('Ignoring SubcloudNotFound when attempting '
'sync_status update: %s' % subcloud_name)
continue
elif audit_fail_count != subcloud.audit_fail_count:
try:
db_api.subcloud_update(self.context, subcloud_id,
management_state=None,
availability_status=None,
software_version=None,
description=None, location=None,
audit_fail_count=audit_fail_count)
except exceptions.SubcloudNotFound:
# slim possibility subcloud could have been deleted since
# we found it in db, ignore this benign error.
LOG.info('Ignoring SubcloudNotFound when attempting '
'audit_fail_count update: %s' % subcloud_name)
continue
if sysinv_client:
# get a list of installed apps in the subcloud
try:
apps = sysinv_client.get_applications()
except Exception as e:
apps = None
LOG.warn('Cannot retrieve installed apps for '
'subcloud:%s, %s' % (subcloud_name, e))
if apps:
openstack_installed_current = False
for app in apps:
if app.name == sysinv_constants.HELM_APP_OPENSTACK\
and app.active:
# audit find openstack app is installed and active in
# the subcloud
openstack_installed_current = True
break
dcm_update_func = None
dco_update_func = None
if openstack_installed_current and not openstack_installed:
dcm_update_func = db_api.subcloud_status_create
dco_update_func = self.dcorch_rpc_client.\
add_subcloud_sync_endpoint_type
elif not openstack_installed_current and openstack_installed:
dcm_update_func = db_api.subcloud_status_delete
dco_update_func = self.dcorch_rpc_client.\
remove_subcloud_sync_endpoint_type
if dcm_update_func and dco_update_func:
endpoint_type_list = dcorch_consts.ENDPOINT_TYPES_LIST_OS
try:
# Notify dcorch to add/remove sync endpoint type list
dco_update_func(self.context, subcloud_name,
endpoint_type_list)
LOG.info('Notifying dcorch, subcloud: %s new sync'
' endpoint: %s' % (subcloud_name,
endpoint_type_list))
# Update subcloud status table by adding/removing
# openstack sync endpoint types.
for endpoint_type in endpoint_type_list:
dcm_update_func(self.context, subcloud_id,
endpoint_type)
# Update openstack_installed of subcloud table
db_api.subcloud_update(
self.context, subcloud_id,
openstack_installed=openstack_installed_current)
except exceptions.SubcloudNotFound:
LOG.info('Ignoring SubcloudNotFound when attempting'
' openstack_installed update: %s'
% subcloud_name)
except Exception as e:
LOG.exception(e)
LOG.warn('Problem informing dcorch of subcloud '
'sync endpoint type change, subcloud: %s'
% subcloud_name)
'sync endpoint type change, subcloud: %s'
% subcloud_name)

49
distributedcloud/dcmanager/tests/base.py
View File

@ -20,8 +20,57 @@
# of an applicable Wind River license agreement.
#
import sqlalchemy
from oslo_config import cfg
from oslo_db import options
from dcmanager.db import api as api
from dcmanager.db.sqlalchemy import api as db_api
from dcmanager.tests import utils
from oslotest import base
get_engine = api.get_engine
# Enable foreign key support in sqlite - see:
# http://docs.sqlalchemy.org/en/latest/dialects/sqlite.html
from sqlalchemy.engine import Engine
from sqlalchemy import event
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
class DCManagerTestCase(base.BaseTestCase):
"""Test case base class for all unit tests."""
def setup_dummy_db(self):
options.cfg.set_defaults(options.database_opts,
sqlite_synchronous=False)
options.set_defaults(cfg.CONF, connection="sqlite://")
engine = get_engine()
db_api.db_sync(engine)
@staticmethod
def reset_dummy_db():
engine = get_engine()
meta = sqlalchemy.MetaData()
meta.reflect(bind=engine)
for table in reversed(meta.sorted_tables):
if table.name == 'migrate_version':
continue
engine.execute(table.delete())
def setUp(self):
super(DCManagerTestCase, self).setUp()
self.setup_dummy_db()
self.addCleanup(self.reset_dummy_db)
self.ctx = utils.dummy_context()

323
distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py
View File

@ -17,66 +17,321 @@
# of an applicable Wind River license agreement.
#
import copy
import mock
import sys
sys.modules['fm_core'] = mock.Mock()
from oslo_config import cfg
from dcmanager.common import consts
from dcmanager.db.sqlalchemy import api as db_api
from dcmanager.manager import subcloud_audit_manager
from dcmanager.manager import subcloud_manager
from dcorch.common import consts as dcorch_consts
from dcmanager.tests import base
from dcmanager.tests import utils
from dcorch.common import messaging as dcorch_messaging
class FakeDCOrchAPI(object):
def __init__(self):
self.update_subcloud_states = mock.MagicMock()
self.add_subcloud_sync_endpoint_type = mock.MagicMock()
CONF = cfg.CONF
FAKE_PROJECT = 'fake_project'
FAKE_REGION = 'fake_region'
NOVA_USAGE = {'ram': 100, 'cores': '50'}
NEUTRON_USAGE = {'port': 10}
CINDER_USAGE = {'volumes': 18}
FAKE_REGION_DICT = {'region1': {'ram': 100},
'region2': {'ram': 200, 'volumes': 500}}
TOTAL_USAGE = {}
TOTAL_USAGE.update(NOVA_USAGE)
TOTAL_USAGE.update(NEUTRON_USAGE)
TOTAL_USAGE.update(CINDER_USAGE)
TASK_TYPE = 'quota_sync'
class FakeServiceGroup(object):
def __init__(self, status, desired_state, service_group_name, uuid,
node_name, state, condition, name):
self.status = status
self.desired_state = desired_state
self.service_group_name = service_group_name
self.uuid = uuid
self.node_name = node_name
self.state = state
self.condition = condition
self.name = name
class FakeApplication(object):
def __init__(self, status, name, manifest_name, active, progress,
app_version, manifest_file):
self.status = status
self.name = name
self.manifest_name = manifest_name
self.active = active
self.progress = progress
self.app_version = app_version
self.manifest_file = manifest_file
FAKE_SERVICE_GROUPS = [
FakeServiceGroup("",
"active",
"distributed-cloud-services",
"b00fd252-5bd7-44b5-bbde-7d525e7125c7",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"storage-monitoring-services",
"5a14a1d1-dac1-48b0-9598-3702e0b0338a",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"storage-services",
"5cbfa903-379f-4329-81b4-2e88acdfa215",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"web-services",
"42829858-008f-4931-94e1-4b86fe31ce3c",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"directory-services",
"74225295-2601-4376-a52c-7cbd149146f6",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"patching-services",
"6870c079-e1c3-4402-b88b-63a5ef06a77a",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"vim-services",
"d8367a52-316e-418b-9211-a13331e073ef",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"cloud-services",
"12682dc0-cef5-427a-b1a6-145cf950b49c",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"controller-services",
"daac63fb-24b3-4cd1-b895-260a32e356ae",
"controller-0",
"active",
"",
"controller"),
FakeServiceGroup("",
"active",
"oam-services",
"4b66913d-98ba-4a4a-86c3-168625f629eb",
"controller-0",
"active",
"",
"controller"),
]
FAKE_APPLICATIONS = [
FakeApplication("applied",
"platform-integ-apps",
"platform-integration-manifest",
True,
"completed",
"1.0-8",
"manifest.yaml"),
FakeApplication("applied",
"stx-openstack",
"stx-openstack-manifest",
True,
"completed",
"1.0-8",
"manifest.yaml"),
]
class FakeSysinvClient(object):
def __init__(self, region, session):
self.get_service_groups_result = FAKE_SERVICE_GROUPS
self.get_applications_result = FAKE_APPLICATIONS
def get_service_groups(self):
return self.get_service_groups_result
def get_applications(self):
return self.get_applications_result
class TestAuditManager(base.DCManagerTestCase):
def setUp(self):
super(TestAuditManager, self).setUp()
self.ctxt = utils.dummy_context()
dcorch_messaging.setup("fake://", optional=True)
@mock.patch.object(subcloud_audit_manager, 'SysinvClient')
@mock.patch.object(subcloud_audit_manager, 'KeystoneClient')
@mock.patch.object(subcloud_audit_manager, 'context')
def test_init(self, mock_context,
mock_keystone_client,
mock_sysinv_client):
mock_context.get_admin_context.return_value = self.ctxt
# Mock the DCOrch API
self.fake_dcorch_api = FakeDCOrchAPI()
p = mock.patch('dcorch.rpc.client.EngineClient')
self.mock_dcorch_api = p.start()
self.mock_dcorch_api.return_value = self.fake_dcorch_api
self.addCleanup(p.stop)
# Mock the SysinvClient
self.fake_sysinv_client = FakeSysinvClient('fake_region',
'fake_session')
p = mock.patch.object(subcloud_audit_manager, 'SysinvClient')
self.mock_sysinv_client = p.start()
self.mock_sysinv_client.return_value = self.fake_sysinv_client
self.addCleanup(p.stop)
# Mock the KeystoneClient
p = mock.patch.object(subcloud_audit_manager, 'KeystoneClient')
self.mock_keystone_client = p.start()
self.addCleanup(p.stop)
# Mock the context
p = mock.patch.object(subcloud_audit_manager, 'context')
self.mock_context = p.start()
self.mock_context.get_admin_context.return_value = self.ctx
self.addCleanup(p.stop)
@staticmethod
def create_subcloud_static(ctxt, **kwargs):
values = {
'name': "subcloud1",
'description': "This is a subcloud",
'location': "This is the location of the subcloud",
'software_version': "10.04",
'management_subnet': "192.168.101.0/24",
'management_gateway_ip': "192.168.101.1",
'management_start_ip': "192.168.101.2",
'management_end_ip': "192.168.101.50",
'systemcontroller_gateway_ip': "192.168.204.101",
'deploy_status': "not-deployed",
'openstack_installed': False,
}
values.update(kwargs)
return db_api.subcloud_create(ctxt, **values)
def test_init(self):
sm = subcloud_manager.SubcloudManager()
am = subcloud_audit_manager.SubcloudAuditManager(subcloud_manager=sm)
self.assertIsNotNone(am)
self.assertEqual('subcloud_audit_manager', am.service_name)
self.assertEqual('localhost', am.host)
self.assertEqual(self.ctxt, am.context)
self.assertEqual(self.ctx, am.context)
@mock.patch.object(subcloud_audit_manager, 'SysinvClient')
@mock.patch.object(subcloud_audit_manager, 'KeystoneClient')
@mock.patch.object(subcloud_audit_manager, 'context')
def test_periodic_subcloud_audit(self, mock_context,
mock_keystone_client,
mock_sysinv_client):
mock_context.get_admin_context.return_value = self.ctxt
def test_periodic_subcloud_audit(self):
mock_sm = mock.Mock()
am = subcloud_audit_manager.SubcloudAuditManager(
subcloud_manager=mock_sm)
am.periodic_subcloud_audit()
def test_audit_subcloud_online(self):
subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
self.assertIsNotNone(subcloud)
mock_sm = mock.Mock()
am = subcloud_audit_manager.SubcloudAuditManager(
subcloud_manager=mock_sm)
# No stx-openstack application
self.fake_sysinv_client.get_application_results = []
# Audit the subcloud
am._audit_subcloud(subcloud.name, audit_openstack=False)
# Verify the subcloud was set to online
self.fake_dcorch_api.update_subcloud_states.assert_called_with(
mock.ANY, 'subcloud1', consts.MANAGEMENT_UNMANAGED,
consts.AVAILABILITY_ONLINE)
# Verify the openstack endpoints were not added
self.fake_dcorch_api.add_subcloud_sync_endpoint_type.\
assert_not_called()
# Verify the subcloud openstack_installed was not updated
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, 'subcloud1')
self.assertEqual(updated_subcloud.openstack_installed, False)
def test_audit_subcloud_offline(self):
subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
self.assertIsNotNone(subcloud)
mock_sm = mock.Mock()
am = subcloud_audit_manager.SubcloudAuditManager(
subcloud_manager=mock_sm)
# Set the subcloud to online
db_api.subcloud_update(
self.ctx, subcloud.id,
availability_status=consts.AVAILABILITY_ONLINE)
# Mark a service group as inactive
self.fake_sysinv_client.get_service_groups_result = \
copy.deepcopy(FAKE_SERVICE_GROUPS)
self.fake_sysinv_client.get_service_groups_result[3].state = 'inactive'
# Audit the subcloud
am._audit_subcloud(subcloud.name, audit_openstack=False)
# Verify the subcloud was not set to offline
self.fake_dcorch_api.update_subcloud_states.assert_not_called()
# Verify the audit_fail_count was updated
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, 'subcloud1')
self.assertEqual(updated_subcloud.audit_fail_count, 1)
# Audit the subcloud again
am._audit_subcloud(subcloud.name, audit_openstack=False)
# Verify the subcloud was set to offline
self.fake_dcorch_api.update_subcloud_states.assert_called_with(
mock.ANY, 'subcloud1', consts.MANAGEMENT_UNMANAGED,
consts.AVAILABILITY_OFFLINE)
# Verify the sublcoud availability was updated
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, 'subcloud1')
self.assertEqual(updated_subcloud.availability_status,
consts.AVAILABILITY_OFFLINE)
def test_audit_subcloud_online_with_openstack(self):
subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
self.assertIsNotNone(subcloud)
mock_sm = mock.Mock()
am = subcloud_audit_manager.SubcloudAuditManager(
subcloud_manager=mock_sm)
# Audit the subcloud
am._audit_subcloud(subcloud.name, audit_openstack=True)
# Verify the subcloud was set to online
self.fake_dcorch_api.update_subcloud_states.assert_called_with(
mock.ANY, 'subcloud1', consts.MANAGEMENT_UNMANAGED,
consts.AVAILABILITY_ONLINE)
# Verify the openstack endpoints were added
self.fake_dcorch_api.add_subcloud_sync_endpoint_type.\
assert_called_with(mock.ANY, 'subcloud1',
dcorch_consts.ENDPOINT_TYPES_LIST_OS)
# Verify the subcloud openstack_installed was updated
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, 'subcloud1')
self.assertEqual(updated_subcloud.openstack_installed, True)

6
distributedcloud/dcorch/common/endpoint_cache.py
View File

@ -58,7 +58,8 @@ class EndpointCache(object):
project_domain_name=cfg.CONF.cache.admin_project_domain_name,
)
self.admin_session = session.Session(
auth=auth, additional_headers=consts.USER_HEADER)
auth=auth, additional_headers=consts.USER_HEADER,
timeout=cfg.CONF.keystone_authtoken.http_connect_timeout)
self.keystone_client = keystone_client.Client(
session=self.admin_session,
region_name=consts.CLOUD_0)
@ -95,7 +96,8 @@ class EndpointCache(object):
project_domain_name=cfg.CONF.cache.admin_project_domain_name,
)
self.admin_session = session.Session(
auth=sc_auth, additional_headers=consts.USER_HEADER)
auth=sc_auth, additional_headers=consts.USER_HEADER,
timeout=cfg.CONF.keystone_authtoken.http_connect_timeout)
self.keystone_client = keystone_client.Client(
session=self.admin_session,
region_name=region_name)

14
distributedcloud/tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = linters,pep8,py27
envlist = pep8,py27
minversion = 2.3
skipsdist = True
@ -159,15 +159,3 @@ deps = {[testenv]deps}
commands =
pylint {posargs} dcmanager dcorch --rcfile=./pylint.rc
[testenv:linters]
basepython = python3
# bashate ignore:
# E006 - accept long lines
# E040 - false positive on |& syntax (new in bash 4)
whitelist_externals = bash
commands =
bash -c "find {toxinidir} \
\( -name .tox -prune \) \
-o -type f -name '*.yaml' \
-print0 | xargs -0 yamllint"