starlingx/distcloud
Code Issues Proposed changes

Merge "Trigger subcloud audits when identity status is updated"

Browse Source
This commit is contained in:
Zuul 2023-05-24 16:34:59 +00:00 committed by Gerrit Code Review
commit e03bca88e4
11 changed files with 159 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
distributedcloud/dccommon/drivers/openstack/sdk_platform.py
View File

@ -1,4 +1,4 @@
# Copyright 2017-2021 Wind River Inc
# Copyright 2017-2023 Wind River Inc
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -94,6 +94,10 @@ class OpenStackDriver(object):
LOG.debug('keystone_client region %s error: %s' %
(region_name, str(exception)))
raise exception
except keystone_exceptions.NotFound as exception:
LOG.debug('keystone_client region %s error: %s' %
(region_name, str(exception)))
raise exception
except Exception as exception:
LOG.error('keystone_client region %s error: %s' %

1
distributedcloud/dcmanager/api/controllers/v1/subclouds.py
View File

@ -1485,6 +1485,7 @@ class SubcloudsController(object):
software_version=payload['software_version'],
management_state=dccommon_consts.MANAGEMENT_UNMANAGED,
deploy_status=consts.DEPLOY_STATE_PRE_INSTALL,
first_identity_sync_complete=False,
data_install=data_install)
self.dcmanager_rpc_client.reinstall_subcloud(

20
distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py
View File

@ -364,7 +364,6 @@ class SubcloudAuditWorkerManager(manager.Manager):
sysinv_client = None
fm_client = None
avail_to_set = dccommon_consts.AVAILABILITY_OFFLINE
try:
os_client = OpenStackDriver(region_name=subcloud_name,
thread_name='subcloud-audit',
@ -382,6 +381,19 @@ class SubcloudAuditWorkerManager(manager.Manager):
LOG.error("Identity or Platform endpoint for online "
"subcloud: %s not found." % subcloud_name)
except keystone_exceptions.NotFound:
if subcloud.first_identity_sync_complete \
and avail_status_current == dccommon_consts.AVAILABILITY_ONLINE:
# The first identity sync is already complete
# Therefore this is an error
LOG.error("Identity or Platform endpoint for online "
"subcloud: %s not found." % subcloud_name)
else:
LOG.debug("Identity or Platform endpoint for %s not "
"found, ignoring for offline "
"subcloud or identity sync not done." % subcloud_name)
return audits_done, failures
except (keystone_exceptions.EndpointNotFound,
keystone_exceptions.ConnectFailure,
IndexError):
@ -455,9 +467,11 @@ class SubcloudAuditWorkerManager(manager.Manager):
availability_status=avail_status_current,
update_state_only=True)
# If subcloud is managed and online, audit additional resources
# If subcloud is managed and online and the identity was synced once,
# audit additional resources
if (subcloud.management_state == dccommon_consts.MANAGEMENT_MANAGED and
avail_to_set == dccommon_consts.AVAILABILITY_ONLINE):
avail_to_set == dccommon_consts.AVAILABILITY_ONLINE and
subcloud.first_identity_sync_complete):
# Get alarm summary and store in db,
if fm_client:
self.alarm_aggr.update_alarm_summary(subcloud_name, fm_client)

6
distributedcloud/dcmanager/db/api.py
View File

@ -1,5 +1,5 @@
# Copyright (c) 2015 Ericsson AB.
# Copyright (c) 2017-2022 Wind River Systems, Inc.
# Copyright (c) 2017-2023 Wind River Systems, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -174,7 +174,7 @@ def subcloud_update(context, subcloud_id, management_state=None,
deploy_status=None, backup_status=None,
backup_datetime=None, error_description=None,
openstack_installed=None, group_id=None,
data_install=None, data_upgrade=None):
data_install=None, data_upgrade=None, first_identity_sync_complete=None):
"""Update a subcloud or raise if it does not exist."""
return IMPL.subcloud_update(context, subcloud_id, management_state,
availability_status, software_version,
@ -182,7 +182,7 @@ def subcloud_update(context, subcloud_id, management_state=None,
management_start_ip, management_end_ip, location,
audit_fail_count, deploy_status, backup_status,
backup_datetime, error_description, openstack_installed,
group_id, data_install, data_upgrade)
group_id, data_install, data_upgrade, first_identity_sync_complete)
def subcloud_bulk_update_by_ids(context, subcloud_ids, update_form):

7
distributedcloud/dcmanager/db/sqlalchemy/api.py
View File

@ -1,5 +1,5 @@
# Copyright (c) 2015 Ericsson AB.
# Copyright (c) 2017-2022 Wind River Systems, Inc.
# Copyright (c) 2017-2023 Wind River Systems, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -382,7 +382,8 @@ def subcloud_update(context, subcloud_id, management_state=None,
openstack_installed=None,
group_id=None,
data_install=None,
data_upgrade=None):
data_upgrade=None,
first_identity_sync_complete=None):
with write_session() as session:
subcloud_ref = subcloud_get(context, subcloud_id)
if management_state is not None:
@ -421,6 +422,8 @@ def subcloud_update(context, subcloud_id, management_state=None,
subcloud_ref.openstack_installed = openstack_installed
if group_id is not None:
subcloud_ref.group_id = group_id
if first_identity_sync_complete is not None:
subcloud_ref.first_identity_sync_complete = first_identity_sync_complete
subcloud_ref.save(session)
return subcloud_ref

30
distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/013_first_identity_sync_complete.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright (c) 2023 Wind River Systems, Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import sqlalchemy
def upgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
subcloud = sqlalchemy.Table('subclouds', meta, autoload=True)
# Add the first_identity_sync_complete column
subcloud.create_column(sqlalchemy.Column('first_identity_sync_complete',
sqlalchemy.Boolean,
default=False))
def downgrade(migrate_engine):
raise NotImplementedError('Database downgrade not supported - '
'would drop all tables')

3
distributedcloud/dcmanager/db/sqlalchemy/models.py
View File

@ -1,5 +1,5 @@
# Copyright (c) 2015 Ericsson AB
# Copyright (c) 2017-2022 Wind River Systems, Inc.
# Copyright (c) 2017-2023 Wind River Systems, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -137,6 +137,7 @@ class Subcloud(BASE, DCManagerBase):
openstack_installed = Column(Boolean, nullable=False, default=False)
systemcontroller_gateway_ip = Column(String(255))
audit_fail_count = Column(Integer)
first_identity_sync_complete = Column(Boolean, default=False)
# multiple subclouds can be in a particular group
group_id = Column(Integer,

9
distributedcloud/dcmanager/manager/subcloud_manager.py
View File

@ -1766,16 +1766,9 @@ class SubcloudManager(manager.Manager):
elif management_state == dccommon_consts.MANAGEMENT_MANAGED:
# Subcloud is managed
# Tell cert-mon to audit endpoint certificate
LOG.info('Request for managed audit for %s' % subcloud.name)
LOG.info('Request certmon audit for %s' % subcloud.name)
dc_notification = dcmanager_rpc_client.DCManagerNotifications()
dc_notification.subcloud_managed(context, subcloud.name)
# Since sysinv user is sync'ed during bootstrap, trigger the
# related audits. Patch and load audits are delayed until the
# identity resource synchronized by dcdbsync is complete.
exclude_endpoints = [dccommon_consts.ENDPOINT_TYPE_PATCHING,
dccommon_consts.ENDPOINT_TYPE_LOAD]
self.audit_rpc_client.trigger_subcloud_audits(
context, subcloud_id, exclude_endpoints)
return db_api.subcloud_db_model_to_dict(subcloud)

12
distributedcloud/dcmanager/state/subcloud_state_manager.py
View File

@ -10,7 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2017-2022 Wind River Systems, Inc.
# Copyright (c) 2017-2023 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
@ -131,15 +131,17 @@ class SubcloudStateManager(manager.Manager):
endpoint_type,
sync_status)
# Trigger subcloud patch and load audits for the subcloud after
# Trigger subcloud audits for the subcloud after
# its identity endpoint turns to other status from unknown
if endpoint_type == dccommon_consts.ENDPOINT_TYPE_IDENTITY \
and sync_status != dccommon_consts.SYNC_STATUS_UNKNOWN \
and original_identity_status == dccommon_consts.SYNC_STATUS_UNKNOWN:
LOG.debug('Request for patch and load audit for %s after updating '
if not subcloud.first_identity_sync_complete:
db_api.subcloud_update(context, subcloud_id,
first_identity_sync_complete=True)
LOG.debug('Request for audits for %s after updating '
'identity out of unknown' % subcloud.name)
self.audit_rpc_client.trigger_subcloud_patch_load_audits(
context, subcloud_id)
self.audit_rpc_client.trigger_subcloud_audits(context, subcloud_id)
entity_instance_id = "subcloud=%s.resource=%s" % \
(subcloud.name, endpoint_type)

82
distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_worker_manager.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2017-2022 Wind River Systems, Inc.
# Copyright (c) 2017-2023 Wind River Systems, Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -391,7 +391,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
# Set the subcloud to managed
subcloud = db_api.subcloud_update(
self.ctx, subcloud.id,
management_state='managed')
management_state='managed',
first_identity_sync_complete=True)
am = subcloud_audit_manager.SubcloudAuditManager()
wm = subcloud_audit_worker_manager.SubcloudAuditWorkerManager()
@ -460,6 +461,78 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
self.fake_kube_rootca_update_audit.subcloud_audit.assert_called_with(
subcloud.name, kube_rootca_update_audit_data)
def test_audit_subcloud_online_first_identity_sync_not_complete(self):
subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
self.assertIsNotNone(subcloud)
# Set the subcloud to managed
subcloud = db_api.subcloud_update(
self.ctx, subcloud.id,
management_state='managed')
am = subcloud_audit_manager.SubcloudAuditManager()
wm = subcloud_audit_worker_manager.SubcloudAuditWorkerManager()
# Audit the subcloud
update_subcloud_state = False
do_audit_openstack = False
do_patch_audit = True
do_load_audit = True
do_firmware_audit = True
do_kubernetes_audit = True
do_kube_rootca_update_audit = True
(patch_audit_data, firmware_audit_data,
kubernetes_audit_data, kube_rootca_update_audit_data) = \
am._get_audit_data(do_patch_audit,
do_firmware_audit,
do_kubernetes_audit,
do_kube_rootca_update_audit)
# Convert to dict like what would happen calling via RPC
# Note: the other data should also be converted...
patch_audit_data = patch_audit_data.to_dict()
wm._audit_subcloud(subcloud,
update_subcloud_state,
do_audit_openstack,
patch_audit_data,
firmware_audit_data,
kubernetes_audit_data,
kube_rootca_update_audit_data,
do_patch_audit,
do_load_audit,
do_firmware_audit,
do_kubernetes_audit,
do_kube_rootca_update_audit)
# Verify the subcloud was set to online
self.fake_dcmanager_state_api.update_subcloud_availability.assert_called_with(
mock.ANY, subcloud.name, dccommon_consts.AVAILABILITY_ONLINE,
False, 0)
# Verify the _update_subcloud_audit_fail_count is not called
with mock.patch.object(wm, '_update_subcloud_audit_fail_count') as \
mock_update_subcloud_audit_fail_count:
mock_update_subcloud_audit_fail_count.assert_not_called()
# Verify the openstack endpoints were not added
self.fake_dcmanager_api.update_subcloud_sync_endpoint_type.\
assert_not_called()
# Verify alarm update is not called
self.fake_alarm_aggr.update_alarm_summary.assert_not_called()
# Verify patch audit is not called
self.fake_patch_audit.subcloud_patch_audit.assert_not_called()
# Verify firmware audit is not called
self.fake_firmware_audit.subcloud_firmware_audit.assert_not_called()
# Verify kubernetes audit is not called
self.fake_kubernetes_audit.subcloud_kubernetes_audit.assert_not_called()
# Verify kube rootca update audit is not called
self.fake_kube_rootca_update_audit.subcloud_audit.assert_not_called()
def test_audit_subcloud_online_unmanaged(self):
subcloud = self.create_subcloud_static(self.ctx, name='subcloud1')
@ -635,6 +708,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
subcloud = db_api.subcloud_update(
self.ctx, subcloud.id,
management_state='managed',
first_identity_sync_complete=True,
availability_status=dccommon_consts.AVAILABILITY_ONLINE)
# Mark a service group as inactive
@ -895,6 +969,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
subcloud = db_api.subcloud_update(
self.ctx, subcloud.id,
management_state='managed',
first_identity_sync_complete=True,
availability_status=dccommon_consts.AVAILABILITY_ONLINE)
# Audit the subcloud
@ -954,6 +1029,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
self.ctx, subcloud.id,
management_state='managed',
availability_status=dccommon_consts.AVAILABILITY_ONLINE,
first_identity_sync_complete=True,
openstack_installed=True)
# Remove stx-openstack application
@ -1014,6 +1090,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
subcloud = db_api.subcloud_update(
self.ctx, subcloud.id,
management_state='managed',
first_identity_sync_complete=True,
availability_status=dccommon_consts.AVAILABILITY_ONLINE,
openstack_installed=True)
@ -1071,6 +1148,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase):
# Set the subcloud to managed
subcloud = db_api.subcloud_update(
self.ctx, subcloud.id,
first_identity_sync_complete=True,
management_state='managed')
am = subcloud_audit_manager.SubcloudAuditManager()

24
distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py
View File

@ -661,11 +661,6 @@ class TestSubcloudManager(base.DCManagerTestCase):
fake_dcmanager_notification.subcloud_managed.assert_called_once_with(
self.ctx, subcloud.name)
exclude_endpoints = [dccommon_consts.ENDPOINT_TYPE_PATCHING,
dccommon_consts.ENDPOINT_TYPE_LOAD]
self.fake_dcmanager_audit_api.trigger_subcloud_audits.\
assert_called_once_with(self.ctx, subcloud.id, exclude_endpoints)
# Verify subcloud was updated with correct values
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(dccommon_consts.MANAGEMENT_MANAGED,
@ -761,10 +756,6 @@ class TestSubcloudManager(base.DCManagerTestCase):
fake_dcmanager_cermon_api.subcloud_managed.assert_called_once_with(
self.ctx, subcloud.name)
exclude_endpoints = [dccommon_consts.ENDPOINT_TYPE_PATCHING,
dccommon_consts.ENDPOINT_TYPE_LOAD]
self.fake_dcmanager_audit_api.trigger_subcloud_audits.\
assert_called_once_with(self.ctx, subcloud.id, exclude_endpoints)
# Verify subcloud was updated with correct values
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
@ -1227,7 +1218,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# We trigger a subcloud audits after updating the identity from unknown
# to in-sync
self.fake_dcmanager_audit_api.trigger_subcloud_patch_load_audits.\
self.fake_dcmanager_audit_api.trigger_subcloud_audits.\
assert_called_once_with(self.ctx, subcloud.id)
# Audit fails once
@ -1283,6 +1274,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# Set the subcloud to online/managed
db_api.subcloud_update(self.ctx, subcloud.id,
management_state=dccommon_consts.MANAGEMENT_MANAGED,
first_identity_sync_complete=True,
availability_status=dccommon_consts.AVAILABILITY_ONLINE)
# Update identity endpoints statuses
@ -1302,19 +1294,19 @@ class TestSubcloudManager(base.DCManagerTestCase):
sync_status=original_sync_status)
# Get the count of the trigger already called
original_trigger_subcloud_patch_load_audits_count = \
self.fake_dcmanager_audit_api.trigger_subcloud_patch_load_audits.call_count
trigger_subcloud_audits = \
self.fake_dcmanager_audit_api.trigger_subcloud_audits.call_count
# Update identity to new status and get the count of the trigger again
ssm.update_subcloud_endpoint_status(
self.ctx, subcloud_name=subcloud.name,
endpoint_type=endpoint,
sync_status=new_sync_status)
new_trigger_subcloud_patch_load_audits_count = \
self.fake_dcmanager_audit_api.trigger_subcloud_patch_load_audits.call_count
new_trigger_subcloud_audits = \
self.fake_dcmanager_audit_api.trigger_subcloud_audits.call_count
trigger_count = new_trigger_subcloud_patch_load_audits_count - \
original_trigger_subcloud_patch_load_audits_count
trigger_count = new_trigger_subcloud_audits - \
trigger_subcloud_audits
if original_sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN and \
new_sync_status != dccommon_consts.SYNC_STATUS_UNKNOWN: