Implement USM software deploy start for upgrade

When a major/minor release update is to deployed, start deploy
start script and execute the deploy start operations in background
process. Once the background subprocess starts, return to user w/
deploy started.

Story: 2010676
Task: 48958

TCs:
    To pass: complete deploy start on controller-0 on a DX system

Depends-on: https://review.opendev.org/c/starlingx/update/+/898850

Change-Id: I5c0317c584348a9c1473719ffd01f810c3121212
Signed-off-by: Bin Qian <bin.qian@windriver.com>

software/software/software_controller.py

@@ -101,6 +101,18 @@ DEPLOY_STATE_METADATA_DIR_DICT = \
 # Limit socket blocking to 5 seconds to allow for thread to shutdown
 api_socket_timeout = 5.0
 
+from packaging import version
+
+# TODO(bqian) Move to utils.py
+def is_upgrade_deploy(from_release, to_release):
+    from_ver = version.Version(from_release)
+    to_ver = version.Version(to_release)
+
+    if from_ver.major == to_ver.major and from_ver.minor == to_ver.minor:
+        return False
+    else:
+        return True
+
 
 class ControllerNeighbour(object):
     def __init__(self):
@@ -1860,6 +1872,61 @@ class PatchController(PatchService):
 
         return dict(info=msg_info, warning=msg_warning, error=msg_error)
 
+    # TODO(bqian): move to utilities functions
+    def get_k8s_ver(self):
+        from software.software_functions import get_sysinv_client
+        from software.software_functions import get_endpoints_token
+
+        try:
+            token, endpoint = get_endpoints_token()
+            sysinv_client = get_sysinv_client(token=token, endpoint=endpoint)
+            k8s_vers = sysinv_client.kube_version.list()
+            if(len(k8s_vers) > 0):
+                return k8s_vers[-1].version
+            else:
+                raise Exception("Failed to get current k8s version")
+        except Exception as err:
+            LOG.error("Error getting k8s version: %s", err)
+            raise
+
+    def _deploy_upgrade_start(self, to_release):
+        LOG.info("start deploy upgrade to %s from %s" % (to_release, SW_VERSION))
+        cmd_path = "/usr/bin/software_deploy_start"
+        k8s_ver = self.get_k8s_ver()
+        postgresql_port = "6666"
+        root_dir = "/sysroot/upgrade/sysroot"
+        ostree_repo_dir = "/sysroot/upgrade/ostree_repo"
+        feed = "/var/www/pages/feed/rel-%s/ostree_repo" % to_release
+        instbr = "starlingx"
+        commit_id = None
+
+        LOG.info("k8s version %s" % k8s_ver)
+        # from_ver to_ver k8s_ver postgresql_port root_dir ostree_repo_dir feed instbr
+        upgrade_start_cmd = [cmd_path, SW_VERSION, to_release, k8s_ver, postgresql_port,
+                             root_dir, ostree_repo_dir, feed, instbr]
+        if commit_id is not None:
+            upgrade_start_cmd.append(commit_id)
+        # pass in keystone auth through environment variables
+        # OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_PROJECT_NAME, OS_USER_DOMAIN_NAME,
+        # OS_PROJECT_DOMAIN_NAME, OS_REGION_NAME are in env variables.
+        keystone_auth = CONF.get('keystone_authtoken')
+        env["OS_AUTH_URL"] = keystone_auth["auth_url"] + '/v3'
+        env["OS_USERNAME"] = keystone_auth["username"]
+        env["OS_PASSWORD"] = keystone_auth["password"]
+        env["OS_PROJECT_NAME"] = keystone_auth["project_name"]
+        env["OS_USER_DOMAIN_NAME"] = keystone_auth["user_domain_name"]
+        env["OS_PROJECT_DOMAIN_NAME"] = keystone_auth["project_domain_name"]
+        env["OS_REGION_NAME"] = keystone_auth["region_name"]
+
+        try:
+            LOG.info("starting subprocess %s" % ' '.join(upgrade_start_cmd))
+            subprocess.Popen(upgrade_start_cmd, env=env)
+            LOG.info("subprocess started")
+            return True
+        except subprocess.CalledProcessError as e:
+            LOG.error("Failed to start command: %s. Error %s" % (' '.join(upgrade_start_cmd), e))
+            return False
+
     def software_deploy_start_api(self, deployment: str, **kwargs) -> dict:
         """
         Start deployment by applying the changes to the feed ostree
@@ -1877,6 +1944,13 @@ class PatchController(PatchService):
             msg_error += msg + "\n"
             return dict(info=msg_info, warning=msg_warning, error=msg_error)
 
+        if is_upgrade_deploy(SW_VERSION, self.release_data.metadata[deployment]["sw_version"]):
+            if self._deploy_upgrade_start(self.release_data.metadata[deployment]["sw_version"]):
+                msg_info = "Deploy %s started" % deployment
+            else:
+                msg_error = "Deploy %s failed to start" % deployment
+
+            return dict(info=msg_info, warning=msg_warning, error=msg_error)
         # Identify if this is apply or remove operation
         # todo(jcasteli) Remove once the logic to include major release version
         # in release list is implemented