Extend sysinv api proxy to support load operations

In this commit, the dcorch-sysinv-api-proxy is extended
to support load import and delete requests.

Upon receiving a successful response from sysinv api for load
import/delete request, the load is saved to/removed from dc-vault
accordingly.

Tests:
  - Successful load import
  - Successful load delete (no subclouds have the deleted load)
  - Successful load delete (one subcloud has the deleted load)
  - Failed load import (exceeding limit)
  - Failed load import (bad signature)
  - Failed load delete (does not exist)

Unit tests will be added via a separate commit under task 39903
of story 2007082.

Story: 2007403
Task: 39840
Depends-On: https://review.opendev.org/#/c/730632
Change-Id: If2769b0faf093523e7e9bc97b8cdc6a5513534aa
Signed-off-by: Tee Ngo <tee.ngo@windriver.com>

distributedcloud/centos/build_srpm.data

@@ -1,4 +1,4 @@
 SRC_DIR="."
 COPY_LIST="$FILES_BASE/*"
 
-TIS_PATCH_VER=3
+TIS_PATCH_VER=PKG_GITREVCOUNT

distributedcloud/dccommon/drivers/openstack/sysinv_v1.py

@@ -20,6 +20,7 @@
 #
 
 import hashlib
+import os
 
 from cgtsclient.exc import HTTPConflict
 from cgtsclient.exc import HTTPNotFound
@@ -29,8 +30,6 @@ from cgtsclient.v1.itrapdest import CREATION_ATTRIBUTES \
     as SNMP_TRAPDEST_CREATION_ATTRIBUTES
 from oslo_log import log
 
-from sysinv.common import constants as sysinv_constants
-
 from dccommon import consts
 from dccommon.drivers import base
 from dccommon import exceptions
@@ -39,6 +38,25 @@ from dccommon import exceptions
 LOG = log.getLogger(__name__)
 API_VERSION = '1'
 
+CERT_CA_FILE = "ca-cert.pem"
+CERT_MODE_DOCKER_REGISTRY = 'docker_registry'
+CERT_MODE_SSL = 'ssl'
+CERT_MODE_SSL_CA = 'ssl_ca'
+CERT_MODE_TPM = 'tpm_mode'
+
+CONTROLLER = 'controller'
+
+NETWORK_TYPE_MGMT = 'mgmt'
+
+SSL_CERT_CA_DIR = "/etc/pki/ca-trust/source/anchors/"
+SSL_CERT_CA_FILE = os.path.join(SSL_CERT_CA_DIR, CERT_CA_FILE)
+SSL_CERT_DIR = "/etc/ssl/private/"
+SSL_CERT_FILE = "server-cert.pem"
+SSL_PEM_FILE = os.path.join(SSL_CERT_DIR, SSL_CERT_FILE)
+
+DOCKER_REGISTRY_CERT_FILE = os.path.join(SSL_CERT_DIR, "registry-cert.crt")
+DOCKER_REGISTRY_KEY_FILE = os.path.join(SSL_CERT_DIR, "registry-cert.key")
+
 
 def make_sysinv_patch(update_dict):
     patch = []
@@ -83,7 +101,7 @@ class SysinvClient(base.DriverBase):
     def get_controller_hosts(self):
         """Get a list of controller hosts."""
         return self.sysinv_client.ihost.list_personality(
-            sysinv_constants.CONTROLLER)
+            CONTROLLER)
 
     def get_management_interface(self, hostname):
         """Get the management interface for a host."""
@@ -92,7 +110,7 @@ class SysinvClient(base.DriverBase):
             interface_networks = self.sysinv_client.interface_network.\
                 list_by_interface(interface.uuid)
             for if_net in interface_networks:
-                if if_net.network_type == sysinv_constants.NETWORK_TYPE_MGMT:
+                if if_net.network_type == NETWORK_TYPE_MGMT:
                     return interface
 
         # This can happen if the host is still being installed and has not
@@ -104,7 +122,7 @@ class SysinvClient(base.DriverBase):
         """Get the management address pool for a host."""
         networks = self.sysinv_client.network.list()
         for network in networks:
-            if network.type == sysinv_constants.NETWORK_TYPE_MGMT:
+            if network.type == NETWORK_TYPE_MGMT:
                 address_pool_uuid = network.pool_uuid
                 break
         else:
@@ -168,6 +186,32 @@ class SysinvClient(base.DriverBase):
         """Get a list of loads."""
         return self.sysinv_client.load.list()
 
+    def get_load(self, load_id):
+        """Get a particular load."""
+        return self.sysinv_client.load.get(load_id)
+
+    def delete_load(self, load_id):
+        """Delete a load with the given id
+
+           :param: load id
+        """
+        try:
+            LOG.info("delete_load region {} load_id: {}".format(
+                     self.region_name, load_id))
+            self.sysinv_client.load.delete(load_id)
+        except HTTPNotFound:
+            LOG.info("delete_load NotFound {} for region: {}".format(
+                     load_id, self.region_name))
+            raise exceptions.LoadNotFound(region_name=self.region_name,
+                                          load_id=load_id)
+        except Exception as e:
+            LOG.error("delete_load exception={}".format(e))
+            raise e
+
+    def get_hosts(self):
+        """Get a list of hosts."""
+        return self.sysinv_client.ihost.list()
+
     def get_upgrades(self):
         """Get a list of upgrades."""
         return self.sysinv_client.upgrade.list()
@@ -425,33 +469,30 @@ class SysinvClient(base.DriverBase):
         if not certificate:
             if data:
                 data['passphrase'] = None
-                mode = data.get('mode', sysinv_constants.CERT_MODE_SSL)
+                mode = data.get('mode', CERT_MODE_SSL)
-                if mode == sysinv_constants.CERT_MODE_SSL_CA:
+                if mode == CERT_MODE_SSL_CA:
-                    certificate_files = [sysinv_constants.SSL_CERT_CA_FILE]
+                    certificate_files = [SSL_CERT_CA_FILE]
-                elif mode == sysinv_constants.CERT_MODE_SSL:
+                elif mode == CERT_MODE_SSL:
-                    certificate_files = [sysinv_constants.SSL_PEM_FILE]
+                    certificate_files = [SSL_PEM_FILE]
-                elif mode == sysinv_constants.CERT_MODE_DOCKER_REGISTRY:
+                elif mode == CERT_MODE_DOCKER_REGISTRY:
                     certificate_files = \
-                        [sysinv_constants.DOCKER_REGISTRY_KEY_FILE,
+                        [DOCKER_REGISTRY_KEY_FILE,
-                         sysinv_constants.DOCKER_REGISTRY_CERT_FILE]
+                         DOCKER_REGISTRY_CERT_FILE]
                 else:
                     LOG.warn("update_certificate mode {} not supported".format(
                         mode))
                     return
-            elif signature and signature.startswith(
+            elif signature and signature.startswith(CERT_MODE_SSL_CA):
-                    sysinv_constants.CERT_MODE_SSL_CA):
+                data['mode'] = CERT_MODE_SSL_CA
-                data['mode'] = sysinv_constants.CERT_MODE_SSL_CA
+                certificate_files = [SSL_CERT_CA_FILE]
-                certificate_files = [sysinv_constants.SSL_CERT_CA_FILE]
+            elif signature and signature.startswith(CERT_MODE_SSL):
-            elif signature and signature.startswith(
+                data['mode'] = CERT_MODE_SSL
-                    sysinv_constants.CERT_MODE_SSL):
+                certificate_files = [SSL_PEM_FILE]
-                data['mode'] = sysinv_constants.CERT_MODE_SSL
+            elif signature and signature.startswith(CERT_MODE_DOCKER_REGISTRY):
-                certificate_files = [sysinv_constants.SSL_PEM_FILE]
+                data['mode'] = CERT_MODE_DOCKER_REGISTRY
-            elif signature and signature.startswith(
-                    sysinv_constants.CERT_MODE_DOCKER_REGISTRY):
-                data['mode'] = sysinv_constants.CERT_MODE_DOCKER_REGISTRY
                 certificate_files = \
-                    [sysinv_constants.DOCKER_REGISTRY_KEY_FILE,
+                    [DOCKER_REGISTRY_KEY_FILE,
-                     sysinv_constants.DOCKER_REGISTRY_CERT_FILE]
+                     DOCKER_REGISTRY_CERT_FILE]
             else:
                 LOG.warn("update_certificate signature {} "
                          "not supported".format(signature))
@@ -466,8 +507,8 @@ class SysinvClient(base.DriverBase):
                 signature, certificate_files))
 
         if (signature and
-                (signature.startswith(sysinv_constants.CERT_MODE_SSL) or
+                (signature.startswith(CERT_MODE_SSL) or
-                    (signature.startswith(sysinv_constants.CERT_MODE_TPM)))):
+                    (signature.startswith(CERT_MODE_TPM)))):
             # ensure https is enabled
             isystem = self.sysinv_client.isystem.list()[0]
             https_enabled = isystem.capabilities.get('https_enabled', False)

distributedcloud/dccommon/exceptions.py

@@ -98,3 +98,7 @@ class CommunityNotFound(NotFound):
 class CertificateNotFound(NotFound):
     message = _("Certificate in region=%(region_name)s with signature "
                 "%(signature)s not found")
+
+
+class LoadNotFound(NotFound):
+    message = _("Load in region=%(region_name)s with id %(load_id)s not found")

distributedcloud/dcmanager/common/consts.py

@@ -138,6 +138,3 @@ DEPLOY_COMMON_FILE_OPTIONS = [
     DEPLOY_OVERRIDES,
     DEPLOY_CHART
 ]
-
-# Active load state
-LOAD_STATE_ACTIVE = 'active'

distributedcloud/dcorch/api/proxy/apps/controller.py

@@ -1,4 +1,4 @@
-# Copyright 2017-2019 Wind River
+# Copyright 2017-2020 Wind River
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,9 +14,14 @@
 # limitations under the License.
 
 import json
+import os
+import shutil
 import webob.dec
 import webob.exc
 
+from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
+from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
+from dcmanager.common import consts as dcmanager_consts
 from dcorch.api.proxy.apps.dispatcher import APIDispatcher
 from dcorch.api.proxy.apps.proxy import Proxy
 from dcorch.api.proxy.common import constants as proxy_consts
@@ -27,11 +32,11 @@ from dcorch.common import consts
 import dcorch.common.context as k_context
 from dcorch.common import exceptions as exception
 from dcorch.common import utils
+from dcorch.rpc import client as rpc_client
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service.wsgi import Request
-
+from oslo_utils._i18n import _
-from dcorch.rpc import client as rpc_client
 
 
 LOG = logging.getLogger(__name__)
@@ -386,10 +391,90 @@ class SysinvAPIController(APIController):
         }
 
     def _process_response(self, environ, request_body, response):
-        if self.get_status_code(response) in self.OK_STATUS_CODE:
+        try:
-            self._enqueue_work(environ, request_body, response)
+            if self.get_status_code(response) in self.OK_STATUS_CODE:
-            self.notify(environ, self.ENDPOINT_TYPE)
+                resource_type = self._get_resource_type_from_environ(environ)
-        return response
+                operation_type = proxy_utils.get_operation_type(environ)
+
+                if resource_type == consts.RESOURCE_TYPE_SYSINV_LOAD:
+                    if operation_type == consts.OPERATION_TYPE_POST:
+                        resp = json.loads(response.body)
+                        if resp.get('error'):
+                            self._check_load_in_vault()
+                        else:
+                            new_load = resp.get('new_load')
+                            self._save_load_to_vault(new_load['software_version'])
+                    else:
+                        sw_version = json.loads(response.body)['software_version']
+                        self._remove_load_from_vault(sw_version)
+
+                else:
+                    self._enqueue_work(environ, request_body, response)
+                    self.notify(environ, self.ENDPOINT_TYPE)
+
+            return response
+        finally:
+            proxy_utils.cleanup(environ)
+
+    def _save_load_to_vault(self, sw_version):
+        versioned_vault = os.path.join(proxy_consts.LOAD_VAULT_DIR,
+                                       sw_version)
+
+        try:
+            if not os.path.isdir(versioned_vault):
+                os.makedirs(versioned_vault)
+
+            # Copy the load files from staging directory
+            load_path = proxy_consts.LOAD_FILES_STAGING_DIR
+            load_files = [f for f in os.listdir(load_path)
+                          if os.path.isfile(os.path.join(load_path, f))]
+            if len(load_files) != len(proxy_consts.IMPORT_LOAD_FILES):
+                msg = _("Failed to store load in vault. Please check "
+                        "dcorch log for details.")
+                raise webob.exc.HTTPInsufficientStorage(explanation=msg)
+
+            for lf in load_files:
+                shutil.copy(os.path.join(load_path, lf), versioned_vault)
+
+            LOG.info("Load (%s) saved to vault." % sw_version)
+        except Exception:
+            msg = _("Failed to store load in vault. Please check "
+                    "dcorch log for details.")
+            raise webob.exc.HTTPInsufficientStorage(explanation=msg)
+
+    def _remove_load_from_vault(self, sw_version):
+        versioned_vault = os.path.join(
+            proxy_consts.LOAD_VAULT_DIR, sw_version)
+
+        if os.path.isdir(versioned_vault):
+            shutil.rmtree(versioned_vault)
+            LOG.info("Load (%s) removed from vault." % sw_version)
+
+    def _check_load_in_vault(self):
+        if not os.path.exists(proxy_consts.LOAD_VAULT_DIR):
+            # The vault directory has not even been created. This must
+            # be the very first load-import request which failed.
+            return
+        elif len(os.listdir(proxy_consts.LOAD_VAULT_DIR)) == 0:
+            try:
+                ks_client = OpenStackDriver(
+                    region_name=dcmanager_consts.DEFAULT_REGION_NAME,
+                    region_clients=None).keystone_client
+                sysinv_client = SysinvClient(
+                    dcmanager_consts.DEFAULT_REGION_NAME, ks_client.session)
+                loads = sysinv_client.get_loads()
+            except Exception:
+                # Shouldn't be here
+                LOG.exception("Failed to get list of loads.")
+                return
+            else:
+                if len(loads) > proxy_consts.IMPORTED_LOAD_MAX_COUNT:
+                    # The previous load regardless of its current state
+                    # was mistakenly imported without the proxy.
+                    msg = _("Previous load was not imported in the right "
+                            "region. Please remove the previous load and "
+                            "re-import it using 'SystemController' region.")
+                    raise webob.exc.HTTPUnprocessableEntity(explanation=msg)
 
     def _enqueue_work(self, environ, request_body, response):
         LOG.info("enqueue_work")
@@ -413,6 +498,10 @@ class SysinvAPIController(APIController):
                                     for res in resource]
                 else:
                     resource_ids = [resource.get('signature')]
+        elif resource_type == consts.RESOURCE_TYPE_SYSINV_LOAD:
+            if operation_type == consts.OPERATION_TYPE_DELETE:
+                resource_id = json.loads(response.body)['software_version']
+                resource_ids = [resource_id]
         else:
             if (operation_type == consts.OPERATION_TYPE_POST and
                     resource_type in self.RESOURCE_ID_MAP):

distributedcloud/dcorch/api/proxy/common/constants.py

@@ -1,4 +1,4 @@
-# Copyright 2017-2019 Wind River
+# Copyright 2017-2020 Wind River
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -99,6 +99,10 @@ USER_PATHS = [
     '/v1/iuser/{uuid}'
 ]
 
+LOAD_PATHS = [
+    '/v1/loads/import_load',
+    '/v1/loads/{id}'
+]
 
 SYSINV_PATH_MAP = {
     consts.RESOURCE_TYPE_SYSINV_DNS: DNS_PATHS,
@@ -106,8 +110,13 @@ SYSINV_PATH_MAP = {
     consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: COMMUNITY_STRING_PATHS,
     consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: CERTIFICATE_PATHS,
     consts.RESOURCE_TYPE_SYSINV_USER: USER_PATHS,
+    consts.RESOURCE_TYPE_SYSINV_LOAD: LOAD_PATHS,
 }
 
+LOAD_FILES_STAGING_DIR = '/scratch/tmp_load'
+IMPORT_LOAD_FILES = ['path_to_iso', 'path_to_sig']
+IMPORTED_LOAD_MAX_COUNT = 1
+
 # Cinder
 CINDER_QUOTA_PATHS = [
     '/{version}/{admin_project_id}/os-quota-sets/{project_id}',
@@ -318,6 +327,7 @@ ROUTE_METHOD_MAP = {
         consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: ['POST', 'DELETE'],
         consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: ['POST', 'DELETE'],
         consts.RESOURCE_TYPE_SYSINV_USER: ['PATCH', 'PUT'],
+        consts.RESOURCE_TYPE_SYSINV_LOAD: ['POST', 'DELETE'],
     },
     consts.ENDPOINT_TYPE_NETWORK: {
         consts.RESOURCE_TYPE_NETWORK_SECURITY_GROUP: ['POST', 'PUT', 'DELETE'],
@@ -354,3 +364,7 @@ ROUTE_METHOD_MAP = {
 
     }
 }
+
+LOAD_VAULT_DIR = '/opt/dc-vault/loads'
+ENDPOINT_TYPE_PATCHING_TMPDIR = "/scratch/patch-api-proxy-tmpdir"
+ENDPOINT_TYPE_PLATFORM_TMPDIR = "/scratch/platform-api-proxy-tmpdir"

distributedcloud/dcorch/cmd/api_proxy.py

@@ -33,6 +33,7 @@ import logging as std_logging
 from dcmanager.common import messaging as dcmanager_messaging
 from dcorch.api import api_config
 from dcorch.api import app
+from dcorch.api.proxy.common import constants
 
 from dcorch.common import config
 from dcorch.common import consts
@@ -66,6 +67,12 @@ CONF.register_cli_opts(proxy_cli_opts)
 LOG = logging.getLogger('dcorch.api.proxy')
 
 
+def make_tempdir(tempdir):
+    if not os.path.isdir(tempdir):
+        os.makedirs(tempdir)
+    os.environ['TMPDIR'] = tempdir
+
+
 def main():
     api_config.init(sys.argv[1:])
     api_config.setup_logging()
@@ -92,14 +99,13 @@ def main():
              {'host': host, 'port': port, 'workers': workers})
     systemd.notify_once()
 
-    # create a temp directory under /scratch and set TMPDIR
+    # For patching and platorm, create a temp directory under /scratch
-    # environment variable to this directory, so that the file created
+    # and set TMPDIR environment variable to this directory, so that
-    # using tempfile will not use the default directory
+    # the file created using tempfile will not use the default directory.
-    if (CONF.type == consts.ENDPOINT_TYPE_PATCHING):
+    if CONF.type == consts.ENDPOINT_TYPE_PATCHING:
-        tempdir = os.path.join('/scratch', 'patch-api-proxy-tmpdir')
+        make_tempdir(constants.ENDPOINT_TYPE_PATCHING_TMPDIR)
-        if not os.path.isdir(tempdir):
+    elif CONF.type == consts.ENDPOINT_TYPE_PLATFORM:
-            os.makedirs(tempdir)
+        make_tempdir(constants.ENDPOINT_TYPE_PLATFORM_TMPDIR)
-        os.environ['TMPDIR'] = tempdir
 
     service = wsgi.Server(CONF, CONF.prog, application, host, port)
 

distributedcloud/dcorch/common/consts.py

@@ -79,6 +79,7 @@ RESOURCE_TYPE_SYSINV_SNMP_COMM = "icommunity"
 RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST = "itrapdest"
 RESOURCE_TYPE_SYSINV_USER = "iuser"
 RESOURCE_TYPE_SYSINV_FERNET_REPO = "fernet_repo"
+RESOURCE_TYPE_SYSINV_LOAD = "loads"
 
 # Compute Resources
 RESOURCE_TYPE_COMPUTE_FLAVOR = "flavor"
@@ -179,3 +180,6 @@ INITIAL_SYNC_STATE_REQUESTED = "requested"
 INITIAL_SYNC_STATE_IN_PROGRESS = "in-progress"
 INITIAL_SYNC_STATE_COMPLETED = "completed"
 INITIAL_SYNC_STATE_FAILED = "failed"
+
+# Active load state
+LOAD_STATE_ACTIVE = 'active'

distributedcloud/ocf/dcorch-sysinv-api-proxy

@@ -30,7 +30,7 @@
 
 OCF_RESKEY_binary_default="/usr/bin/dcorch-api-proxy"
 OCF_RESKEY_config_default="/etc/dcorch/dcorch.conf"
-OCF_RESKEY_user_default="dcorch"
+OCF_RESKEY_user_default="root"
 OCF_RESKEY_pid_default="$HA_RSCTMP/$OCF_RESOURCE_INSTANCE.pid"
 
 : ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}}