Software deploy host validation
This commit validates pre-conditions when software deploy-host <hostname> is issued. The pre-conditions are: The host state is in pending state. The specified host target is locked and online. Nodes deployed to major release in the order below in DX system: Controller-1 -> Controller-0 -> Storage nodes -> Compute nodes. Nodes deployed to patch release in the order below in DX system: Controllers -> Storages nodes -> Compute nodes. Test Plan: PASS: Deploy host to controller-1 validation. PASS: Validated target locked and online checker. PASS: Validated start-done deploy state checker. Story: 2010676 Task: 49795 Change-Id: I8fe8faa85c594472bb6c8c021416205bf4a61fbb Signed-off-by: Luis Eduardo Bonatti <LuizEduardo.Bonatti@windriver.com>
This commit is contained in:
parent
24016bd363
commit
f99f8377d0
@ -19,6 +19,8 @@ from tsconfig.tsconfig import SW_VERSION
|
|||||||
ADDRESS_VERSION_IPV4 = 4
|
ADDRESS_VERSION_IPV4 = 4
|
||||||
ADDRESS_VERSION_IPV6 = 6
|
ADDRESS_VERSION_IPV6 = 6
|
||||||
CONTROLLER_FLOATING_HOSTNAME = "controller"
|
CONTROLLER_FLOATING_HOSTNAME = "controller"
|
||||||
|
CONTROLLER_0_HOSTNAME = '%s-0' % CONTROLLER_FLOATING_HOSTNAME
|
||||||
|
CONTROLLER_1_HOSTNAME = '%s-1' % CONTROLLER_FLOATING_HOSTNAME
|
||||||
|
|
||||||
DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER = 'systemcontroller'
|
DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER = 'systemcontroller'
|
||||||
SYSTEM_CONTROLLER_REGION = 'SystemController'
|
SYSTEM_CONTROLLER_REGION = 'SystemController'
|
||||||
@ -74,6 +76,7 @@ PATCH_EXTENSION = ".patch"
|
|||||||
SUPPORTED_UPLOAD_FILE_EXT = [ISO_EXTENSION, SIG_EXTENSION, PATCH_EXTENSION]
|
SUPPORTED_UPLOAD_FILE_EXT = [ISO_EXTENSION, SIG_EXTENSION, PATCH_EXTENSION]
|
||||||
SCRATCH_DIR = "/scratch"
|
SCRATCH_DIR = "/scratch"
|
||||||
RELEASE_GA_NAME = "starlingx-%s"
|
RELEASE_GA_NAME = "starlingx-%s"
|
||||||
|
MAJOR_RELEASE = "%s.0"
|
||||||
|
|
||||||
# Precheck constants
|
# Precheck constants
|
||||||
LICENSE_FILE = "/etc/platform/.license"
|
LICENSE_FILE = "/etc/platform/.license"
|
||||||
@ -94,3 +97,11 @@ LAST_IN_SYNC = "last_in_sync"
|
|||||||
|
|
||||||
SYSTEM_MODE_SIMPLEX = "simplex"
|
SYSTEM_MODE_SIMPLEX = "simplex"
|
||||||
SYSTEM_MODE_DUPLEX = "duplex"
|
SYSTEM_MODE_DUPLEX = "duplex"
|
||||||
|
|
||||||
|
# Personalities
|
||||||
|
CONTROLLER = 'controller'
|
||||||
|
STORAGE = 'storage'
|
||||||
|
WORKER = 'worker'
|
||||||
|
|
||||||
|
AVAILABILITY_ONLINE = 'online'
|
||||||
|
ADMIN_LOCKED = 'locked'
|
||||||
|
@ -55,6 +55,7 @@ from software.release_data import reload_release_data
|
|||||||
from software.release_data import get_SWReleaseCollection
|
from software.release_data import get_SWReleaseCollection
|
||||||
from software.software_functions import collect_current_load_for_hosts
|
from software.software_functions import collect_current_load_for_hosts
|
||||||
from software.software_functions import create_deploy_hosts
|
from software.software_functions import create_deploy_hosts
|
||||||
|
from software.software_functions import deploy_host_validations
|
||||||
from software.software_functions import parse_release_metadata
|
from software.software_functions import parse_release_metadata
|
||||||
from software.software_functions import configure_logging
|
from software.software_functions import configure_logging
|
||||||
from software.software_functions import mount_iso_load
|
from software.software_functions import mount_iso_load
|
||||||
@ -757,7 +758,7 @@ class SWMessageDeployStateChanged(messages.PatchMessage):
|
|||||||
(self.deploy_state, self.agent))
|
(self.deploy_state, self.agent))
|
||||||
sc.deploy_state_changed(self.deploy_state)
|
sc.deploy_state_changed(self.deploy_state)
|
||||||
else:
|
else:
|
||||||
LOG.info("Received %s deploy state changed to %s, agent %s" %
|
LOG.info("Received %s deploy host state changed to %s, agent %s" %
|
||||||
(self.hostname, self.host_state, self.agent))
|
(self.hostname, self.host_state, self.agent))
|
||||||
sc.host_deploy_state_changed(self.hostname, self.host_state)
|
sc.host_deploy_state_changed(self.hostname, self.host_state)
|
||||||
|
|
||||||
@ -2715,6 +2716,7 @@ class PatchController(PatchService):
|
|||||||
if deploy_host is None:
|
if deploy_host is None:
|
||||||
raise HostNotFound(hostname)
|
raise HostNotFound(hostname)
|
||||||
|
|
||||||
|
deploy_host_validations(hostname)
|
||||||
deploy_state = DeployState.get_instance()
|
deploy_state = DeployState.get_instance()
|
||||||
deploy_host_state = DeployHostState(hostname)
|
deploy_host_state = DeployHostState(hostname)
|
||||||
deploy_state.deploy_host()
|
deploy_state.deploy_host()
|
||||||
|
@ -210,7 +210,7 @@ class DeployHosts(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def update(self, hostname: str, state: str):
|
def update(self, hostname: str, state: DEPLOY_HOST_STATES):
|
||||||
"""
|
"""
|
||||||
Update a deploy-host entry
|
Update a deploy-host entry
|
||||||
|
|
||||||
|
@ -39,6 +39,8 @@ import software.constants as constants
|
|||||||
from software import states
|
from software import states
|
||||||
import software.utils as utils
|
import software.utils as utils
|
||||||
from software.sysinv_utils import get_ihost_list
|
from software.sysinv_utils import get_ihost_list
|
||||||
|
from software.sysinv_utils import get_system_info
|
||||||
|
from software.sysinv_utils import is_host_locked_and_online
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -1215,14 +1217,17 @@ def create_deploy_hosts():
|
|||||||
Create deploy-hosts entities based on hostnames
|
Create deploy-hosts entities based on hostnames
|
||||||
from sysinv.
|
from sysinv.
|
||||||
"""
|
"""
|
||||||
try:
|
|
||||||
db_api_instance = get_instance()
|
db_api_instance = get_instance()
|
||||||
|
db_api_instance.begin_update()
|
||||||
|
try:
|
||||||
for ihost in get_ihost_list():
|
for ihost in get_ihost_list():
|
||||||
db_api_instance.create_deploy_host(ihost.hostname)
|
db_api_instance.create_deploy_host(ihost.hostname)
|
||||||
LOG.info("Deploy-hosts entities created successfully.")
|
LOG.info("Deploy-hosts entities created successfully.")
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
LOG.exception("Error in deploy-hosts entities creation")
|
LOG.exception("Error in deploy-hosts entities creation")
|
||||||
raise err
|
raise err
|
||||||
|
finally:
|
||||||
|
db_api_instance.end_update()
|
||||||
|
|
||||||
|
|
||||||
def collect_current_load_for_hosts():
|
def collect_current_load_for_hosts():
|
||||||
@ -1330,3 +1335,86 @@ def set_host_target_load(hostname, major_release):
|
|||||||
LOG.exception("Error setting target_load to %s for %s: %s" % (
|
LOG.exception("Error setting target_load to %s for %s: %s" % (
|
||||||
major_release, hostname, str(e)))
|
major_release, hostname, str(e)))
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def validate_host_state_to_deploy_host(hostname):
|
||||||
|
"""
|
||||||
|
Check if the deployment host state for the hostname is pending.
|
||||||
|
|
||||||
|
If the validation fails raise SoftwareServiceError exception.
|
||||||
|
|
||||||
|
:param hostname: Hostname of the host to be deployed
|
||||||
|
"""
|
||||||
|
|
||||||
|
host_state = get_instance().get_deploy_host_by_hostname(hostname).get("state")
|
||||||
|
if host_state != states.DEPLOY_HOST_STATES.PENDING.value:
|
||||||
|
msg = (f"Host state is {host_state} and should be "
|
||||||
|
f"{states.DEPLOY_HOST_STATES.PENDING.value}")
|
||||||
|
raise SoftwareServiceError(msg)
|
||||||
|
|
||||||
|
def deploy_host_validations(hostname):
|
||||||
|
"""
|
||||||
|
Check the conditions below:
|
||||||
|
Host state is pending.
|
||||||
|
If system mode is duplex, check if provided hostname satisfy the right deployment order.
|
||||||
|
Host is locked and online.
|
||||||
|
|
||||||
|
If one of the validations fail, raise SoftwareServiceError exception, except if system
|
||||||
|
is a simplex.
|
||||||
|
|
||||||
|
:param hostname: Hostname of the host to be deployed
|
||||||
|
"""
|
||||||
|
validate_host_state_to_deploy_host(hostname)
|
||||||
|
_, system_mode = get_system_info()
|
||||||
|
simplex = (system_mode == constants.SYSTEM_MODE_SIMPLEX)
|
||||||
|
if simplex:
|
||||||
|
LOG.info("System mode is simplex. Skipping deploy order validation...")
|
||||||
|
else:
|
||||||
|
validate_host_deploy_order(hostname)
|
||||||
|
if not is_host_locked_and_online(hostname):
|
||||||
|
msg = f"Host {hostname} must be {constants.ADMIN_LOCKED}."
|
||||||
|
raise SoftwareServiceError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_host_deploy_order(hostname):
|
||||||
|
"""
|
||||||
|
Check if the host to be deployed satisfy the major release deployment right
|
||||||
|
order of controller-1 -> controller-0 -> storages -> computes
|
||||||
|
and for patch release: controllers -> storages -> computes
|
||||||
|
|
||||||
|
Case one of the validations failed raise SoftwareError exception
|
||||||
|
|
||||||
|
:param hostname: Hostname of the host to be deployed.
|
||||||
|
"""
|
||||||
|
db_api_instance = get_instance()
|
||||||
|
controllers_list = [constants.CONTROLLER_1_HOSTNAME, constants.CONTROLLER_0_HOSTNAME]
|
||||||
|
storage_list = []
|
||||||
|
workers_list = []
|
||||||
|
is_patch_release = False
|
||||||
|
deploy = db_api_instance.get_deploy_all()[0]
|
||||||
|
to_release = deploy.get("from_release")
|
||||||
|
if to_release != (constants.MAJOR_RELEASE % utils.get_major_release_version(to_release)):
|
||||||
|
is_patch_release = True
|
||||||
|
for host in get_ihost_list():
|
||||||
|
if host.personality == constants.STORAGE:
|
||||||
|
storage_list.append(host.hostname)
|
||||||
|
if host.personality == constants.WORKER:
|
||||||
|
workers_list.append(host.hostname)
|
||||||
|
|
||||||
|
ordered_storage_list = sorted(storage_list, key=lambda x: int(x.split("-")[1]))
|
||||||
|
ordered_list = controllers_list + ordered_storage_list + workers_list
|
||||||
|
|
||||||
|
for host in db_api_instance.get_deploy_host():
|
||||||
|
if host.get("state") == states.DEPLOY_HOST_STATES.DEPLOYED.value:
|
||||||
|
ordered_list.remove(host.get("hostname"))
|
||||||
|
if not ordered_list:
|
||||||
|
raise SoftwareServiceError("All hosts are already in deployed state.")
|
||||||
|
# If there is only workers nodes there is no order to deploy
|
||||||
|
if hostname == ordered_list[0] or (ordered_list[0] in workers_list and hostname in workers_list):
|
||||||
|
return
|
||||||
|
# If deployment is a patch release bypass the controllers order
|
||||||
|
elif is_patch_release and ordered_list[0] in controllers_list and hostname in controllers_list:
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
raise SoftwareServiceError(f"{hostname.capitalize()} do not satisfy the right order of deployment "
|
||||||
|
f"should be {ordered_list[0]}")
|
||||||
|
@ -54,6 +54,22 @@ def get_ihost_list():
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def is_host_locked_and_online(host):
|
||||||
|
for ihost in get_ihost_list():
|
||||||
|
if (host == ihost.hostname and ihost.availability == constants.AVAILABILITY_ONLINE and
|
||||||
|
ihost.administrative == constants.ADMIN_LOCKED):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_system_info():
|
||||||
|
"""Returns system type and system mode"""
|
||||||
|
token, endpoint = utils.get_endpoints_token()
|
||||||
|
sysinv_client = get_sysinv_client(token=token, endpoint=endpoint)
|
||||||
|
system_info = sysinv_client.isystem.list()[0]
|
||||||
|
return system_info.system_type, system_info.system_mode
|
||||||
|
|
||||||
|
|
||||||
def get_dc_role():
|
def get_dc_role():
|
||||||
try:
|
try:
|
||||||
token, endpoint = utils.get_endpoints_token()
|
token, endpoint = utils.get_endpoints_token()
|
||||||
|
@ -4,8 +4,14 @@
|
|||||||
# Copyright (c) 2024 Wind River Systems, Inc.
|
# Copyright (c) 2024 Wind River Systems, Inc.
|
||||||
#
|
#
|
||||||
import unittest
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from software import states
|
||||||
|
from software.exceptions import SoftwareServiceError
|
||||||
from software.release_data import SWReleaseCollection
|
from software.release_data import SWReleaseCollection
|
||||||
from software.software_functions import ReleaseData
|
from software.software_functions import ReleaseData
|
||||||
|
from software.software_functions import validate_host_state_to_deploy_host
|
||||||
|
|
||||||
metadata = """<?xml version="1.0" ?>
|
metadata = """<?xml version="1.0" ?>
|
||||||
<patch>
|
<patch>
|
||||||
@ -185,3 +191,17 @@ class TestSoftwareFunction(unittest.TestCase):
|
|||||||
self.assertEqual(val["restart_script"], r.restart_script)
|
self.assertEqual(val["restart_script"], r.restart_script)
|
||||||
self.assertEqual(val["commit_id"], r.commit_id)
|
self.assertEqual(val["commit_id"], r.commit_id)
|
||||||
self.assertEqual(val["checksum"], r.commit_checksum)
|
self.assertEqual(val["checksum"], r.commit_checksum)
|
||||||
|
|
||||||
|
|
||||||
|
@patch('software.db.api.SoftwareAPI')
|
||||||
|
def test_validate_host_state_to_deploy_host_raises_exception_if_deploy_host_state_is_wrong(self, software_api_mock):
|
||||||
|
# Arrange
|
||||||
|
deploy_host_state = states.DEPLOY_HOST_STATES.DEPLOYED.value
|
||||||
|
deploy_by_hostname = MagicMock(return_value={"state": deploy_host_state})
|
||||||
|
software_api_mock.return_value = MagicMock(get_deploy_host_by_hostname=deploy_by_hostname)
|
||||||
|
with self.assertRaises(SoftwareServiceError) as error:
|
||||||
|
# Actions
|
||||||
|
validate_host_state_to_deploy_host(hostname="abc")
|
||||||
|
# Assertions
|
||||||
|
error_msg = "Host state is deployed and should be pending"
|
||||||
|
self.assertEqual(str(error.exception), error_msg)
|
||||||
|
@ -53,7 +53,7 @@ def update_deploy_state(server_addr, server_port, agent, deploy_state=None, host
|
|||||||
"agent": agent,
|
"agent": agent,
|
||||||
"deploy-state": deploy_state,
|
"deploy-state": deploy_state,
|
||||||
"hostname": host,
|
"hostname": host,
|
||||||
"host_state": host_state
|
"host-state": host_state
|
||||||
}
|
}
|
||||||
|
|
||||||
msg_txt = json.dumps(msg)
|
msg_txt = json.dumps(msg)
|
||||||
|
Loading…
Reference in New Issue
Block a user