starlingx
/
config
Code Issues Proposed changes

Preventing unlock and swact while k8s rootca update 

To avoid some possible errors tracking procedure states we decided to
block unlock and swact host actions while certain phases of the
procedure are being executed.
For the reboot case we are considering that the option, in case of a
failure due the host action is going to be an abort command execution
and a new whole cycle of k8s rootca update procedure to set the
cluster in an expected state.
For this matter we've added couple methods that will validate
the host actions mentioned above.

Test Plan:
Swact
PASS: prevent swact during host update update-certs phase
PASS: execute swact during kube rootca procedure but without any
      phase being executed
PASS: execute swact after generate certs and proceed until the end of
      the procedure
PASS: prevent swact during pods update

Unlock
PASS: Lock host after start and unlock host after update-cert.
      Then execute procedure until the end.
PASS: Lock host after start and prevent unlock during k8s rootca
      update update-certs phase
PASS: Lock host after start and prevent unlock during pods update
      phase trust-both-cas
PASS: Lock and unlock host without k8s rootca update procedure
      ongoing
PASS: Lock host while update phase in progress

Reboot
PASS: Cold shutdown host being updated. After turn on, abort
      procedure and complete a full cycle

Closes-Bug: 1945329
Signed-off-by: Joao Soubihe <JoaoPaulo.Soubihe@windriver.com>
Change-Id: Ifdd9bbf5802caba244d14e9c1de59cc71828ad04
This commit is contained in:
Joao Soubihe 2021-10-05 10:33:37 -03:00
parent 6c521a6fdd
commit 6f74b61518
2 changed files with 172 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py
View File

@ -3513,6 +3513,46 @@ class HostController(rest.RestController):
"Wait for kubelet upgrade to complete." % ihost['hostname'])
raise wsme.exc.ClientSideError(msg)
def _semantic_check_unlock_kube_rootca_update(self, ihost, force_unlock=False):
"""
Perform semantic checks related to kubernetes rootca update
prior to unlocking host.
"""
if force_unlock:
LOG.warning("Host %s force unlock while kubernetes "
"rootca update in progress." % ihost['hostname'])
return
try:
kube_rootca_update = \
pecan.request.dbapi.kube_rootca_update_get_one()
if kube_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS,
kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA]:
msg = _("Can not unlock %s while kubernetes root ca "
"update phase in progress. Wait for update "
"phase to complete." % ihost['hostname'])
raise wsme.exc.ClientSideError(msg)
except exception.NotFound:
LOG.debug("No kubernetes rootca update was found")
return
try:
kube_host_rootca_update = \
pecan.request.dbapi.kube_rootca_host_update_get_by_host(ihost['uuid'])
if kube_host_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS,
kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS,
kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA]:
msg = _("Can not unlock %s while kubernetes root ca "
"update phase in progress. Wait for update "
"phase to complete on host." % ihost['hostname'])
raise wsme.exc.ClientSideError(msg)
except exception.NotFound:
LOG.debug("No kubernetes rootca update on host %s "
"was found" % ihost['hostname'])
return
def _semantic_check_unlock_upgrade(self, ihost, force_unlock=False):
"""
Perform semantic checks related to upgrades prior to unlocking host.
@ -5376,6 +5416,10 @@ class HostController(rest.RestController):
# the unlock.
self.check_unlock_application(hostupdate, force_unlock)
# Ensure there is no k8s rootca update phase in progress
self._semantic_check_unlock_kube_rootca_update(hostupdate.ihost_orig,
force_unlock)
personality = hostupdate.ihost_patch.get('personality')
if personality == constants.CONTROLLER:
self.check_unlock_controller(hostupdate, force_unlock)
@ -6037,6 +6081,47 @@ class HostController(rest.RestController):
# Check for new hardware since upgrade-start
self._semantic_check_upgrade_refresh(upgrade, to_host, force_swact)
def _semantic_check_swact_kube_rootca_update(self, ihost, force_swact=False):
"""
Perform semantic checks related to kubernetes rootca update
prior to swacting host.
"""
if force_swact:
LOG.warning("Host %s force swact while kubernetes "
"rootca update in progress on host."
% ihost['hostname'])
return
try:
kube_rootca_update = \
pecan.request.dbapi.kube_rootca_update_get_one()
if kube_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS,
kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA]:
msg = _("Can not swact %s while kubernetes root ca "
"update phase in progress. Wait for update "
"phase to complete." % ihost['hostname'])
raise wsme.exc.ClientSideError(msg)
except exception.NotFound:
LOG.debug("No kubernetes rootca update was found")
return
try:
kube_host_rootca_update = \
pecan.request.dbapi.kube_rootca_host_update_get_by_host(ihost['uuid'])
if kube_host_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS,
kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS,
kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA]:
msg = _("Can not swact %s while kubernetes root ca "
"update phase is in progress. Wait for update "
"phase to complete on host." % ihost['hostname'])
raise wsme.exc.ClientSideError(msg)
except exception.NotFound:
LOG.debug("No kubernetes rootca update on host %s "
"was found" % ihost['hostname'])
return
def _check_swact_device_image_update(self, from_host, to_host, force=False):
if force:
LOG.info("device image update swact check bypassed with force option")
@ -6114,6 +6199,10 @@ class HostController(rest.RestController):
ihost_ctr.subfunction_oper,
ihost_ctr.subfunctions))
# deny swact if a kube rootca update phase is in progress
self._semantic_check_swact_kube_rootca_update(hostupdate.ihost_orig,
force_swact)
# deny swact if storage backend not ready
self._semantic_check_storage_backend(ihost_ctr)

83
sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py
View File

@ -1995,6 +1995,27 @@ class TestPatch(TestHost):
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertTrue(response.json['error_message'])
def test_unlock_action_controller_during_k8s_rootca_pods_update(self):
# Create controller-0 without inv_state initial inventory complete
c0_host = self._create_controller_0(
invprovision=constants.PROVISIONED,
administrative=constants.ADMIN_LOCKED,
operational=constants.OPERATIONAL_ENABLED,
availability=constants.AVAILABILITY_ONLINE,
inv_state=None, clock_synchronization=constants.NTP)
# Create kube rootca update updating pods on phase trust-both-cas
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS)
# Unlock host
response = self._patch_host_action(c0_host['hostname'],
constants.UNLOCK_ACTION,
'sysinv-test',
expect_errors=True)
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertTrue(response.json['error_message'])
def _test_lock_action_controller(self):
# Create controller-0
self._create_controller_0(
@ -2790,6 +2811,68 @@ class TestPatchStdDuplexControllerAction(TestHost):
"images." % (c0_host['hostname'], c1_host['hostname']),
response.json['error_message'])
def test_swact_action_controller_while_kube_rootca_pods_update(self):
# Create controller-0
c0_host = self._create_controller_0(
invprovision=constants.PROVISIONED,
administrative=constants.ADMIN_UNLOCKED,
operational=constants.OPERATIONAL_ENABLED,
availability=constants.AVAILABILITY_ONLINE)
self._create_controller_1(
invprovision=constants.PROVISIONED,
administrative=constants.ADMIN_UNLOCKED,
operational=constants.OPERATIONAL_ENABLED,
availability=constants.AVAILABILITY_ONLINE)
# Create kube rootca update updating pods on phase trust-both-cas
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS)
# Swact controller host
response = self._patch_host_action(c0_host['hostname'],
constants.SWACT_ACTION,
'sysinv-test',
expect_errors=True)
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertTrue(response.json['error_message'])
self.assertIn("Can not swact %s while kubernetes root ca "
"update phase in progress. Wait for update "
"phase to complete." % c0_host['hostname'],
response.json['error_message'])
def test_swact_action_controller_while_kube_rootca_host_update(self):
# Create controller-0
c0_host = self._create_controller_0(
invprovision=constants.PROVISIONED,
administrative=constants.ADMIN_UNLOCKED,
operational=constants.OPERATIONAL_ENABLED,
availability=constants.AVAILABILITY_ONLINE)
self._create_controller_1(
invprovision=constants.PROVISIONED,
administrative=constants.ADMIN_UNLOCKED,
operational=constants.OPERATIONAL_ENABLED,
availability=constants.AVAILABILITY_ONLINE)
# Create kubernetes rootca update for the host and set it with phase in progress
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS)
dbutils.create_test_kube_rootca_host_update(host_id=c0_host['id'],
state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS)
# Swact controller host
response = self._patch_host_action(c0_host['hostname'],
constants.SWACT_ACTION,
'sysinv-test',
expect_errors=True)
self.assertEqual(response.content_type, 'application/json')
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertTrue(response.json['error_message'])
self.assertIn("Can not swact %s while kubernetes root ca "
"update phase is in progress. Wait for update "
"phase to complete on host." % c0_host['hostname'],
response.json['error_message'])
class TestPatchStdDuplexControllerVIM(TestHost):