Add host command support for the edgeworker node

This is an experimental feature in stx5.0.

This commit enables following commands for the edgeworker node:
  system host-add/system host-update/system host-delete
After the host being added/updated, the mgmt ip of an edgeworker
node will be assigned during the configuration process of it.

There will be limitations of edgeworker nodes before the final
phase of the feature finished:
- The Kubernetes provisioning requires ansible playbook triggered
  manually.
- Gather node HW information is not supported.
- Configure node from controller is not supported.
- Manage node lifecycle is not supported.
- Update/upgrade node is not supported.

Story: 2008129
Task: 40862

Change-Id: I7e6de65ba848d9468a4e5afddd16b1cd9e3cd7dd
Signed-off-by: Mingyuan Qi <mingyuan.qi@intel.com>
Depends-On: https://review.opendev.org/c/starlingx/config/+/761716

sysinv/cgts-client/cgts-client/cgtsclient/v1/iHost_shell.py

@@ -158,7 +158,7 @@ def do_kube_host_upgrade_list(cc, args):
            help='Hostname of the host')
 @utils.arg('-p', '--personality',
            metavar='<personality>',
-           choices=['controller', 'worker', 'storage', 'network', 'profile'],
+           choices=['controller', 'worker', 'edgeworker', 'storage', 'network', 'profile'],
            help='Personality or type of host [REQUIRED]')
 @utils.arg('-s', '--subfunctions',
            metavar='<subfunctions>',

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py

@@ -2621,6 +2621,11 @@ class HostController(rest.RestController):
         loads = pecan.request.dbapi.load_get_list()
         new_target_load = cutils.get_imported_load(loads)
         rpc_ihost = objects.host.get_by_uuid(pecan.request.context, uuid)
+
+        if rpc_ihost.personality == constants.EDGEWORKER:
+            raise wsme.exc.ClientSideError(_(
+                "host-upgrade rejected: Not supported for EDGEWORKER node."))
+
         simplex = (utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX)
         # If this is a simplex system skip this check, there's no other nodes
         if simplex:
@@ -2701,6 +2706,10 @@ class HostController(rest.RestController):
         new_target_load = cutils.get_active_load(loads)
         rpc_ihost = objects.host.get_by_uuid(pecan.request.context, uuid)
 
+        if rpc_ihost.personality == constants.EDGEWORKER:
+            raise wsme.exc.ClientSideError(_(
+                "host-downgrade rejected: Not supported for EDGEWORKER node."))
+
         disable_storage_monitor = False
 
         simplex = (utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX)
@@ -2994,7 +3003,9 @@ class HostController(rest.RestController):
 
     def _validate_hostname(self, hostname, personality):
 
-        if personality and personality == constants.WORKER:
+        if personality and \
+                (personality == constants.WORKER or
+                 personality == constants.EDGEWORKER):
             # Fix of invalid hostnames
             err_tl = 'Name restricted to at most 255 characters.'
             err_ic = 'Name may only contain letters, ' \
@@ -5029,6 +5040,9 @@ class HostController(rest.RestController):
                 cutils.is_aio_duplex_system(pecan.request.dbapi):
             return
 
+        if personality == constants.EDGEWORKER:
+            return
+
         if (utils.SystemHelper.get_product_build() ==
                     constants.TIS_AIO_BUILD):
             msg = _("Personality [%s] for host is not compatible "

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/upgrade.py

@@ -323,9 +323,9 @@ class UpgradeController(rest.RestController):
                     "upgrade-activate rejected: "
                     "Upgrade already activating or activated."))
 
-            hosts = pecan.request.dbapi.ihost_get_list()
-            # All hosts must be unlocked and enabled, and running the new
-            # release
+            # All hosts must be unlocked and enabled,
+            # and running the new release
+            hosts = cutils.get_upgradable_hosts(pecan.request.dbapi)
             for host in hosts:
                 if host['administrative'] != constants.ADMIN_UNLOCKED or \
                         host['operational'] != constants.OPERATIONAL_ENABLED:
@@ -399,8 +399,8 @@ class UpgradeController(rest.RestController):
 
         elif upgrade.state in [constants.UPGRADE_ABORTING,
                                constants.UPGRADE_ABORTING_ROLLBACK]:
-            # All hosts must be running the old release
-            hosts = pecan.request.dbapi.ihost_get_list()
+            # All upgradable hosts must be running the old release
+            hosts = cutils.get_upgradable_hosts(pecan.request.dbapi)
             for host in hosts:
                 host_upgrade = objects.host_upgrade.get_by_host_id(
                     pecan.request.context, host.id)

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -115,8 +115,9 @@ CONFIG_ACTIONS = [SUBFUNCTION_CONFIG_ACTION,
 CONTROLLER = 'controller'
 STORAGE = 'storage'
 WORKER = 'worker'
+EDGEWORKER = 'edgeworker'
 
-PERSONALITIES = [CONTROLLER, STORAGE, WORKER]
+PERSONALITIES = [CONTROLLER, STORAGE, WORKER, EDGEWORKER]
 
 # SUBFUNCTION FEATURES
 SUBFUNCTIONS = 'subfunctions'

sysinv/sysinv/sysinv/sysinv/common/health.py

@@ -267,7 +267,7 @@ class Health(object):
         :param alarm_ignore_list: list of alarm ids to ignore when performing
                                   a health check
         """
-        hosts = self._dbapi.ihost_get_list()
+        hosts = utils.get_upgradable_hosts(self._dbapi)
         output = _('System Health:\n')
         health_ok = True
 

sysinv/sysinv/sysinv/sysinv/common/utils.py

@@ -2471,3 +2471,15 @@ def generate_random_password(length=16):
     if six.PY2:
         password = password.decode()
     return password
+
+
+def get_upgradable_hosts(dbapi):
+    """
+    Get hosts that could be upgraded.
+    """
+    all_hosts = dbapi.ihost_get_list()
+    # TODO:(mingyuan) Exclude edgeworker host from upgradable hosts
+    # until the final phase of the edgeworker feature completed
+    hosts = [i for i in all_hosts if i.personality != constants.EDGEWORKER]
+
+    return hosts

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -1674,6 +1674,18 @@ class ConductorManager(service.PeriodicService):
         # Set up the PXE config file for this host so it can run the installer
         self._update_pxe_config(host)
 
+    def _configure_edgeworker_host(self, context, host):
+        """Configure an edgeworker host with the supplied data.
+
+        Does the following tasks:
+        - Create or update entries in address table
+        - Allocates management address if none exists
+
+        :param context: request context
+        :param host: host object
+        """
+        self._allocate_addresses_for_host(context, host)
+
     def _configure_storage_host(self, context, host):
         """Configure a storage ihost with the supplied data.
 
@@ -1774,6 +1786,13 @@ class ConductorManager(service.PeriodicService):
         self._remove_pxe_config(host)
         self._remove_ceph_mon(host)
 
+    def _unconfigure_edgeworker_host(self, host):
+        """Unconfigure an edgeworker host.
+
+        :param host: a host object.
+        """
+        self._remove_addresses_for_host(host)
+
     def _unconfigure_storage_host(self, host):
         """Unconfigure a storage host.
 
@@ -1809,6 +1828,8 @@ class ConductorManager(service.PeriodicService):
             self._configure_controller_host(context, host)
         elif host.personality == constants.WORKER:
             self._configure_worker_host(context, host)
+        elif host.personality == constants.EDGEWORKER:
+            self._configure_edgeworker_host(context, host)
         elif host.personality == constants.STORAGE:
             self._configure_storage_host(context, host)
         else:
@@ -1844,6 +1865,8 @@ class ConductorManager(service.PeriodicService):
                 self._unconfigure_controller_host(ihost_obj)
             elif personality == constants.WORKER:
                 self._unconfigure_worker_host(ihost_obj, is_cpe)
+            elif personality == constants.EDGEWORKER:
+                self._unconfigure_edgeworker_host(ihost_obj)
             elif personality == constants.STORAGE:
                 self._unconfigure_storage_host(ihost_obj)
             else:
@@ -8135,7 +8158,9 @@ class ConductorManager(service.PeriodicService):
 
     def update_security_feature_config(self, context):
         """Update the kernel options configuration"""
-        personalities = constants.PERSONALITIES
+        # Move the edgeworker personality out since it is not configured by puppet
+        personalities = [i for i in constants.PERSONALITIES if i != constants.EDGEWORKER]
+
         config_uuid = self._config_update_hosts(context, personalities, reboot=True)
 
         config_dict = {

sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py

@@ -313,6 +313,30 @@ class TestPostWorkerMixin(object):
         self.assertEqual(ndict['serialid'], result['serialid'])
 
 
+class TestPostEdgeworkerMixin(object):
+
+    def setUp(self):
+        super(TestPostEdgeworkerMixin, self).setUp()
+
+    def test_create_host_worker(self):
+        # Test creation of worker
+        ndict = dbutils.post_get_test_ihost(hostname='edgeworker-0',
+                                            personality='edgeworker',
+                                            subfunctions=None,
+                                            mgmt_ip=None,
+                                            serialid='serial2',
+                                            bm_ip="128.224.150.195")
+        self.post_json('/ihosts', ndict,
+                       headers={'User-Agent': 'sysinv-test'})
+
+        # Verify that the host was configured
+        self.fake_conductor_api.configure_ihost.assert_called_once()
+        # Verify that the host was created and some basic attributes match
+        result = self.get_json('/ihosts/%s' % ndict['hostname'])
+        self.assertEqual(ndict['personality'], result['personality'])
+        self.assertEqual(ndict['serialid'], result['serialid'])
+
+
 class TestPostKubeUpgrades(TestHost):
 
     def setUp(self):
@@ -3042,6 +3066,11 @@ class PostWorkerHostTestCase(TestPostWorkerMixin, TestHost,
     pass
 
 
+class PostEdgeworkerHostTestCase(TestPostEdgeworkerMixin, TestHost,
+                             dbbase.ControllerHostTestCase):
+    pass
+
+
 class PostAIOHostTestCase(TestPostControllerMixin, TestHost,
                           dbbase.AIOHostTestCase):
     pass

sysinv/sysinv/sysinv/sysinv/tests/base.py

@@ -81,6 +81,26 @@ class Database(fixtures.Fixture):
 
     def post_migrations(self):
         """Any addition steps that are needed outside of the migrations."""
+        # This is a workaround for unit test only.
+        # The migration of adding edgeworker personality works with postgres
+        # db. But sqlite db which is used by unit test neither supports
+        # ALTER TYPE to introduce a new personality, nor supports adding
+        # a new CHECK contraint to an existing table. This implements the
+        # migration of version 109 to add an edgeworker personality enum
+        # to i_host table.
+        personality_check_old = "CHECK (personality IN ('controller', " + \
+            "'worker', 'network', 'storage', 'profile', 'reserve1', " + \
+            "'reserve2'))"
+        personality_check_new = "CHECK (personality IN ('controller', " + \
+            "'worker', 'network', 'storage', 'profile', 'reserve1', " + \
+            "'reserve2', 'edgeworker'))"
+        results = self.engine.execute("SELECT sql FROM sqlite_master \
+            WHERE type='table' AND name='i_host'")
+        create_i_host = results.first().values()[0]
+        create_i_host = create_i_host.replace(personality_check_old,
+                                               personality_check_new)
+        self.engine.execute("ALTER TABLE i_host RENAME TO i_host_bak")
+        self.engine.execute(create_i_host)
 
 
 class ReplaceModule(fixtures.Fixture):