Create an API to reinstall a subcloud

This commit creates a REST API to reinstall a subcloud.

In SubcloudController, the tasks includes:
1. Read the subcloud info from dcmanager db, subclouds table.
2. JSONify the data_install from db, generate the new install values.

In SubcloudManager:
1. Check the subcloud availability, software version before reinstall
2. Check the image value, if image doesn't exists, update the image with
iso in dc-vault
3. Run the install command and apply command

API format: PATCH /v1.0/subclouds/<subcloud>/reinstall
-----
Tests:
-----
Happy path:
1. dcmanager subcloud add: a new subcloud with bootstrap values,
deploy config and install values.
2. dcmanager subcloud delete: an existing subcloud.
3. dcmanager subcloud reconfig: an existing subcloud with deploy
config.
4. dcmanager subcloud reinstall: an existing offline subcloud, this
subcloud has image path in data_install in db dcmanager. After the
tasks of reinstall and bootstrap succeed, reconfig this subcloud with
proper deploy config, this subcloud will be online.
5. upload an image to dc-vault using: system --os-region-name
SystemController load-import -a <bootimage.iso> <bootimage.sig>. Then
using dcmanager subcloud reinstall to an existing offline subcloud,
this subcloud has no image path in data_install in db dcmanager. After
the tasks of reinstall and bootstrap succeed, reconfig this subcloud
with proper deploy config, this subcloud will be online.

Unhappy path:
1. dcmanager subcloud reinstall: an existing online subcloud,
reinstall fails.
2. dcmanager subcloud reinstall: an existing subcloud without
data_install value in db, reinstall fails.
3. dcmanager subcloud reinstall: an existing subcloud with
data_install in db, but missing mandatory install value, reinstall
fails.
4. dcmanager subcloud reinstall: an existing subcloud, but its sw
version in db doesn't meet the sw version with system controllers',
reinstall fails.
5. dcmanager subcloud reinstall: an existing subcloud with
data_install in db, but has no image path in data_install, and also
has no right versioned image in dc-vault in system controller,
reinstall fails.

Story: 2007267
Task: 40732
Change-Id: I8be6d8d11e6b4ee02bbcca499ba8869ba76bffaa
Signed-off-by: Yuxing Jiang <yuxing.jiang@windriver.com>

api-ref/source/api-ref-dcmanager-v1.rst

@@ -407,6 +407,8 @@ internalServerError (500), serviceUnavailable (503)
    "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud."
    "oam_floating_ip (Optional)", "plain", "xsd:string", "OAM Floating IP of the subcloud."
    "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group."
+   "data_install (Optional)", "plain", "xsd:string", "The values of the subcloud installation."
+   "data_upgrade (Optional)", "plain", "xsd:string", "The values of the subcloud upgrade."
 
 ::
 
@@ -446,7 +448,9 @@ internalServerError (500), serviceUnavailable (503)
      "group_id": 1,
      "id": 1,
      "name": "subcloud6",
-     "oam_floating_ip" "10.10.10.12"
+     "oam_floating_ip": "10.10.10.12",
+     "data_install": "{"bootstrap_interface": "eno1", "bootstrap_address": ...}",
+     "data_upgrade": null
    }
 
 This operation does not accept a request body.
@@ -612,6 +616,83 @@ Accepts Content-Type multipart/form-data
      "name": "subcloud6"
    }
 
+**********************************
+Reinstalls a specific subcloud
+**********************************
+
+.. rest_method:: PATCH /v1.0/subclouds/{subcloud}/reinstall
+
+Reinstall and bootstrap a subcloud based on its previous install configurations.
+After reinstall, a reconfigure operation with deploy_config file is expected to deploy the subcloud.
+
+**Normal response codes**
+
+200
+
+**Error response codes**
+
+badRequest (400), unauthorized (401), forbidden (403), badMethod (405),
+HTTPUnprocessableEntity (422), internalServerError (500),
+serviceUnavailable (503)
+
+**Request parameters**
+
+.. csv-table::
+         :header: "Parameter", "Style", "Type", "Description"
+   :widths: 20, 20, 20, 60
+
+   "subcloud", "URI", "xsd:string", "The subcloud reference, name or id."
+
+**Response parameters**
+
+.. csv-table::
+   :header: "Parameter", "Style", "Type", "Description"
+   :widths: 20, 20, 20, 60
+
+   "id", "plain", "xsd:int", "The unique identifier for this object."
+   "created_at", "plain", "xsd:dateTime", "The time when the object was created."
+   "updated_at", "plain", "xsd:dateTime", "The time when the object was last updated."
+   "name", "plain", "xsd:string", "The name provisioned for the subcloud."
+   "description(Optional)", "plain", "xsd:string", "The description of the subcloud."
+   "location(Optional)", "plain", "xsd:string", "The location of the subcloud."
+   "software-version", "plain", "xsd:string", "The software version of the subcloud."
+   "deploy_status", "plain", "xsd:string", "The deployment status of the subcloud."
+   "management-state", "plain", "xsd:string", "Management state of the subcloud."
+   "availability-status (Optional)", "plain", "xsd:string", "Availability status of the subcloud."
+   "management-subnet", "plain", "xsd:string", "Management subnet for subcloud in CIDR format."
+   "management-start-ip", "plain", "xsd:string", "Start of management IP address range for subcloud."
+   "management-end-ip", "plain", "xsd:string", "End of management IP address range for subcloud."
+   "systemcontroller-gateway-ip", "plain", "xsd:string", "Systemcontroller gateway IP Address."
+   "openstack-installed (Optional)", "plain", "xsd:boolean", "Whether openstack is installed on the subcloud."
+   "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group."
+   "data_install", "plain", "xsd:string", "The values of the subcloud installation."
+   "data_upgrade (Optional)", "plain", "xsd:string", "The values of the subcloud upgrade."
+
+::
+
+   {
+     "description": "subcloud description",
+     "management-start-ip": "192.168.204.50",
+     "created-at": "2018-02-25T19:06:35.208505",
+     "updated-at": "2018-02-25T23:01:17.490090",
+     "software-version": "20.06",
+     "management-state": "unmanaged",
+     "availability-status": "offline",
+     "openstack-installed": false,
+     "deploy-status": "pre-install",
+     "systemcontroller-gateway-ip": "192.168.204.101",
+     "location": "location",
+     "management-subnet": "192.168.204.0/24",
+     "management-gateway-ip": "192.168.204.1",
+     "management-end-ip": "192.168.204.100",
+     "group_id": 2,
+     "id": 1,
+     "name": "subcloud6",
+     "data_install": "{"bootstrap_interface": "eno1", "bootstrap_address": ...}",
+     "data_upgrade": null,
+     "deploy_status": "pre-deploy"
+   }
+
 *****************************
 Deletes a specific subcloud
 *****************************

distributedcloud/dcmanager/api/controllers/v1/subclouds.py

@@ -126,15 +126,8 @@ class SubcloudsController(object):
             file_item.file.seek(0, os.SEEK_SET)
             contents = file_item.file.read()
             # the deploy config needs to upload to the override location
-            fn = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, payload['name']
+            fn = self._get_config_file_path(payload['name'], consts.DEPLOY_CONFIG)
-                              + '_deploy_config.yml')
+            self._upload_config_file(contents, fn, consts.DEPLOY_CONFIG)
-            try:
-                with open(fn, "w") as f:
-                    f.write(contents)
-            except Exception:
-                msg = _("Failed to upload %s file" % consts.DEPLOY_CONFIG)
-                LOG.exception(msg)
-                pecan.abort(400, msg)
             payload.update({consts.DEPLOY_CONFIG: fn})
             self._get_common_deploy_files(payload)
 
@@ -173,8 +166,16 @@ class SubcloudsController(object):
                     payload.update({f: data})
         return payload
 
-    @staticmethod
+    def _upload_config_file(self, file_item, config_file, config_type):
-    def _get_reconfig_payload(request, subcloud_name):
+        try:
+            with open(config_file, "w") as f:
+                f.write(file_item)
+        except Exception:
+            msg = _("Failed to upload %s file" % config_type)
+            LOG.exception(msg)
+            pecan.abort(400, msg)
+
+    def _get_reconfig_payload(self, request, subcloud_name):
         payload = dict()
         multipart_data = decoder.MultipartDecoder(request.body,
                                                   pecan.request.headers.get('Content-Type'))
@@ -183,22 +184,62 @@ class SubcloudsController(object):
             for part in multipart_data.parts:
                 header = part.headers.get('Content-Disposition')
                 if filename in header:
-                    file_item = part.content
+                    fn = self._get_config_file_path(subcloud_name, consts.DEPLOY_CONFIG)
-                    fn = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, subcloud_name
+                    self._upload_config_file(part.content, fn, consts.DEPLOY_CONFIG)
-                                      + '_deploy_config.yml')
-                    try:
-                        with open(fn, "w") as f:
-                            f.write(file_item)
-                    except Exception:
-                        msg = _("Failed to upload %s file" % consts.DEPLOY_CONFIG)
-                        LOG.exception(msg)
-                        pecan.abort(400, msg)
                     payload.update({consts.DEPLOY_CONFIG: fn})
                 elif "sysadmin_password" in header:
                     payload.update({'sysadmin_password': part.content})
-        SubcloudsController._get_common_deploy_files(payload)
+        self._get_common_deploy_files(payload)
         return payload
 
+    def _get_config_file_path(self, subcloud_name, config_file_type=None):
+        if config_file_type == consts.DEPLOY_CONFIG:
+            file_path = os.path.join(
+                consts.ANSIBLE_OVERRIDES_PATH,
+                subcloud_name + '_' + config_file_type + '.yml'
+            )
+        elif config_file_type == INSTALL_VALUES:
+            file_path = os.path.join(
+                consts.ANSIBLE_OVERRIDES_PATH + '/' + subcloud_name,
+                config_file_type + '.yml'
+            )
+        else:
+            file_path = os.path.join(
+                consts.ANSIBLE_OVERRIDES_PATH,
+                subcloud_name + '.yml'
+            )
+        return file_path
+
+    def _get_subcloud_db_install_values(self, subcloud):
+        if not subcloud.data_install:
+            msg = _("Failed to read data install from db")
+            LOG.exception(msg)
+            pecan.abort(400, msg)
+
+        install_values = json.loads(subcloud.data_install)
+
+        # mandatory bootstrap parameters
+        mandatory_bootstrap_parameters = [
+            'bootstrap_interface',
+            'bootstrap_address',
+            'bootstrap_address_prefix',
+            'bmc_username',
+            'bmc_address',
+            'bmc_password',
+        ]
+        for p in mandatory_bootstrap_parameters:
+            if p not in install_values:
+                msg = _("Failed to get %s from data_install" % p)
+                LOG.exception(msg)
+                pecan.abort(400, msg)
+
+        install_values.update({
+            'ansible_become_pass': consts.TEMP_SYSADMIN_PASSWORD,
+            'ansible_ssh_pass': consts.TEMP_SYSADMIN_PASSWORD
+        })
+
+        return install_values
+
     @staticmethod
     def _get_updatestatus_payload(request):
         """retrieve payload of a patch request for update_status
@@ -802,7 +843,8 @@ class SubcloudsController(object):
             except RemoteError as e:
                 pecan.abort(422, e.value)
             except Exception:
-                LOG.exception("Unable to create subcloud %s" % name)
+                LOG.exception(
+                    "Unable to create subcloud %s" % payload.get('name'))
                 pecan.abort(500, _('Unable to create subcloud'))
         else:
             pecan.abort(400, _('Invalid request'))
@@ -921,6 +963,30 @@ class SubcloudsController(object):
             except Exception:
                 LOG.exception("Unable to reconfigure subcloud %s" % subcloud.name)
                 pecan.abort(500, _('Unable to reconfigure subcloud'))
+        elif verb == "reinstall":
+            install_values = self._get_subcloud_db_install_values(subcloud)
+            payload = db_api.subcloud_db_model_to_dict(subcloud)
+            for k in ['data_install', 'data_upgrade', 'created-at', 'updated-at']:
+                if k in payload:
+                    del payload[k]
+
+            payload.update({
+                'bmc_password': install_values.get('bmc_password'),
+                'install_values': install_values,
+            })
+
+            try:
+                self.rpc_client.reinstall_subcloud(
+                    context, subcloud_id, payload)
+
+                # Return deploy_status as pre-install
+                subcloud.deploy_status = consts.DEPLOY_STATE_PRE_INSTALL
+                return db_api.subcloud_db_model_to_dict(subcloud)
+            except RemoteError as e:
+                pecan.abort(422, e.value)
+            except Exception:
+                LOG.exception("Unable to reinstall subcloud %s" % subcloud.name)
+                pecan.abort(500, _('Unable to reinstall subcloud'))
         elif verb == 'update_status':
             res = self.updatestatus(subcloud.name)
             return res

distributedcloud/dcmanager/manager/service.py

@@ -142,6 +142,14 @@ class DCManagerService(service.Service):
                                                           subcloud_id,
                                                           payload)
 
+    @request_context
+    def reinstall_subcloud(self, context, subcloud_id, payload):
+        # Reinstall a subcloud
+        LOG.info("Handling reinstall_subcloud request for: %s" % subcloud_id)
+        return self.subcloud_manager.reinstall_subcloud(context,
+                                                        subcloud_id,
+                                                        payload)
+
     @request_context
     def update_subcloud_endpoint_status(self, context, subcloud_name=None,
                                         endpoint_type=None,

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -114,13 +114,6 @@ class SubcloudManager(manager.Manager):
         self.dcorch_rpc_client = dcorch_rpc_client.EngineClient()
         self.fm_api = fm_api.FaultAPIs()
 
-    @staticmethod
-    def _get_ansible_inventory_filename(subcloud_name):
-        ansible_inventory_filename = os.path.join(
-            consts.ANSIBLE_OVERRIDES_PATH,
-            subcloud_name + INVENTORY_FILE_POSTFIX)
-        return ansible_inventory_filename
-
     @staticmethod
     def _get_subcloud_cert_name(subcloud_name):
         cert_name = "%s-adminep-ca-certificate" % subcloud_name
@@ -186,6 +179,44 @@ class SubcloudManager(manager.Manager):
 
         raise Exception("Secret for certificate %s is not ready." % cert_name)
 
+    def _get_ansible_filename(self, subcloud_name, postfix='.yml'):
+        ansible_filename = os.path.join(
+            consts.ANSIBLE_OVERRIDES_PATH,
+            subcloud_name + postfix)
+        return ansible_filename
+
+    def compose_install_command(self, subcloud_name, ansible_subcloud_inventory_file):
+        install_command = [
+            "ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
+            "-i", ansible_subcloud_inventory_file,
+            "--limit", subcloud_name,
+            "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
+                  subcloud_name + '/' + "install_values.yml"]
+        return install_command
+
+    def compose_apply_command(self, subcloud_name, ansible_subcloud_inventory_file):
+        apply_command = [
+            "ansible-playbook", ANSIBLE_SUBCLOUD_PLAYBOOK, "-i",
+            ansible_subcloud_inventory_file,
+            "--limit", subcloud_name
+        ]
+        # Add the overrides dir and region_name so the playbook knows
+        # which overrides to load
+        apply_command += [
+            "-e", str("override_files_dir='%s' region_name=%s") % (
+                consts.ANSIBLE_OVERRIDES_PATH, subcloud_name)]
+        return apply_command
+
+    def compose_deploy_command(self, subcloud_name, ansible_subcloud_inventory_file, payload):
+        deploy_command = [
+            "ansible-playbook", payload[consts.DEPLOY_PLAYBOOK],
+            "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
+                  subcloud_name + '_deploy_values.yml',
+            "-i", ansible_subcloud_inventory_file,
+            "--limit", subcloud_name
+            ]
+        return deploy_command
+
     def add_subcloud(self, context, payload):
         """Add subcloud and notify orchestrators.
 
@@ -207,8 +238,8 @@ class SubcloudManager(manager.Manager):
 
         try:
             # Ansible inventory filename for the specified subcloud
-            ansible_subcloud_inventory_file = SubcloudManager.\
+            ansible_subcloud_inventory_file = self._get_ansible_filename(
-                _get_ansible_inventory_filename(subcloud.name)
+                subcloud.name, INVENTORY_FILE_POSTFIX)
 
             # Create a new route to this subcloud on the management interface
             # on both controllers.
@@ -349,13 +380,10 @@ class SubcloudManager(manager.Manager):
             deploy_command = None
             if "deploy_playbook" in payload:
                 self._prepare_for_deployment(payload, subcloud.name)
-                deploy_command = [
+                deploy_command = self.compose_deploy_command(
-                    "ansible-playbook", payload[consts.DEPLOY_PLAYBOOK],
+                    subcloud.name,
-                    "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
+                    ansible_subcloud_inventory_file,
-                          subcloud.name + "_deploy_values.yml",
+                    payload)
-                    "-i", ansible_subcloud_inventory_file,
-                    "--limit", subcloud.name
-                ]
 
             del payload['sysadmin_password']
             payload['users'] = dict()
@@ -375,28 +403,14 @@ class SubcloudManager(manager.Manager):
             # NOTE: This file should not be deleted if subcloud add fails
             # as it is used for debugging
             self._write_subcloud_ansible_config(context, payload)
-
             install_command = None
             if "install_values" in payload:
-                install_command = [
+                install_command = self.compose_install_command(
-                    "ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
+                    subcloud.name,
-                    "-i", ansible_subcloud_inventory_file,
+                    ansible_subcloud_inventory_file)
-                    "--limit", subcloud.name,
+            apply_command = self.compose_apply_command(
-                    "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
+                subcloud.name,
-                          payload['name'] + '/' + "install_values.yml"
+                ansible_subcloud_inventory_file)
-                ]
-
-            apply_command = [
-                "ansible-playbook", ANSIBLE_SUBCLOUD_PLAYBOOK, "-i",
-                ansible_subcloud_inventory_file,
-                "--limit", subcloud.name
-            ]
-
-            # Add the overrides dir and region_name so the playbook knows
-            # which overrides to load
-            apply_command += [
-                "-e", str("override_files_dir='%s' region_name=%s") % (
-                    consts.ANSIBLE_OVERRIDES_PATH, subcloud.name)]
 
             apply_thread = threading.Thread(
                 target=self.run_deploy,
@@ -427,22 +441,18 @@ class SubcloudManager(manager.Manager):
             deploy_status=consts.DEPLOY_STATE_PRE_DEPLOY)
         try:
             # Ansible inventory filename for the specified subcloud
-            ansible_subcloud_inventory_file = SubcloudManager.\
+            ansible_subcloud_inventory_file = self._get_ansible_filename(
-                _get_ansible_inventory_filename(subcloud.name)
+                subcloud.name, INVENTORY_FILE_POSTFIX)
 
             deploy_command = None
             if "deploy_playbook" in payload:
                 self._prepare_for_deployment(payload, subcloud.name)
-                deploy_command = [
+                deploy_command = self.compose_deploy_command(
-                    "ansible-playbook", payload[consts.DEPLOY_PLAYBOOK],
+                    subcloud.name,
-                    "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
+                    ansible_subcloud_inventory_file,
-                          subcloud.name + "_deploy_values.yml",
+                    payload)
-                    "-i", ansible_subcloud_inventory_file,
-                    "--limit", subcloud.name
-                    ]
 
             del payload['sysadmin_password']
-
             apply_thread = threading.Thread(
                 target=self.run_deploy,
                 args=(subcloud, payload, context, None, None, deploy_command))
@@ -456,6 +466,82 @@ class SubcloudManager(manager.Manager):
                 context, subcloud_id,
                 deploy_status=consts.DEPLOY_STATE_DEPLOY_PREP_FAILED)
 
+    def reinstall_subcloud(self, context, subcloud_id, payload):
+        """Reinstall subcloud
+
+        :param context: request context object
+        :param subcloud_id: subcloud id from db
+        :param payload: subcloud reinstall
+        """
+
+        # Retrieve the subcloud details from the database
+        subcloud = db_api.subcloud_get(context, subcloud_id)
+
+        # Semantic checking
+        if subcloud.availability_status == \
+                consts.AVAILABILITY_ONLINE:
+            raise exceptions.SubcloudNotOffline()
+
+        software_version = str(payload['install_values'].get('software_version'))
+        LOG.info("The type of sw version is %s" % type(SW_VERSION))
+
+        if software_version != SW_VERSION:
+            raise exceptions.BadRequest(
+                resource='subcloud',
+                msg='Software version should match the system controller')
+
+        if 'image' not in payload['install_values']:
+            matching_iso, matching_sig = utils.get_vault_load_files(
+                SW_VERSION)
+            payload['install_values'].update({'image': matching_iso})
+
+        LOG.info("Reinstalling subcloud %s." % subcloud_id)
+
+        subcloud = db_api.subcloud_update(
+            context, subcloud_id,
+            software_version=SW_VERSION,
+            deploy_status=consts.DEPLOY_STATE_PRE_INSTALL)
+
+        try:
+            ansible_subcloud_inventory_file = self._get_ansible_filename(
+                subcloud.name, INVENTORY_FILE_POSTFIX)
+
+            payload['admin_password'] = str(
+                keyring.get_password('CGCS', 'admin'))
+            payload['ansible_become_pass'] = payload['admin_password']
+            payload['ansible_ssh_pass'] = payload['admin_password']
+            payload['install_values']['ansible_ssh_pass'] = \
+                payload['admin_password']
+            payload['install_values']['ansible_become_pass'] = \
+                payload['admin_password']
+            payload['bootstrap-address'] = \
+                payload['install_values']['bootstrap_address']
+
+            utils.create_subcloud_inventory(payload,
+                                            ansible_subcloud_inventory_file)
+
+            self._create_intermediate_ca_cert(payload)
+
+            install_command = self.compose_install_command(
+                subcloud.name,
+                ansible_subcloud_inventory_file)
+            apply_command = self.compose_apply_command(
+                subcloud.name,
+                ansible_subcloud_inventory_file)
+            apply_thread = threading.Thread(
+                target=self.run_deploy,
+                args=(subcloud, payload, context,
+                      install_command, apply_command, None))
+            apply_thread.start()
+            return db_api.subcloud_db_model_to_dict(subcloud)
+        except Exception:
+            LOG.exception("Failed to reinstall subcloud %s" % subcloud.name)
+            # If we failed to reinstall the subcloud, update the
+            # deployment status
+            db_api.subcloud_update(
+                context, subcloud_id,
+                deploy_status=consts.DEPLOY_STATE_DEPLOY_PREP_FAILED)
+
     @staticmethod
     def run_deploy(subcloud, payload, context,
                    install_command=None, apply_command=None,
@@ -753,9 +839,8 @@ class SubcloudManager(manager.Manager):
             raise exceptions.SubcloudNotOffline()
 
         # Ansible inventory filename for the specified subcloud
-        ansible_subcloud_inventory_file = os.path.join(
+        ansible_subcloud_inventory_file = self._get_ansible_filename(
-            consts.ANSIBLE_OVERRIDES_PATH,
+            subcloud.name, INVENTORY_FILE_POSTFIX)
-            subcloud.name + INVENTORY_FILE_POSTFIX)
 
         self._remove_subcloud_details(context,
                                       subcloud,

distributedcloud/dcmanager/rpc/client.py

@@ -100,6 +100,11 @@ class ManagerClient(RPCClient):
                                              subcloud_id=subcloud_id,
                                              payload=payload))
 
+    def reinstall_subcloud(self, ctxt, subcloud_id, payload):
+        return self.cast(ctxt, self.make_msg('reinstall_subcloud',
+                                             subcloud_id=subcloud_id,
+                                             payload=payload))
+
     def update_subcloud_endpoint_status(self, ctxt, subcloud_name=None,
                                         endpoint_type=None,
                                         sync_status=consts.

distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py

@@ -25,6 +25,7 @@ from oslo_utils import timeutils
 import base64
 import copy
 import json
+import keyring
 import mock
 import six
 from six.moves import http_client
@@ -32,44 +33,25 @@ import webtest
 
 from dcmanager.api.controllers.v1 import subclouds
 from dcmanager.common import consts
+from dcmanager.common import utils as cutils
 from dcmanager.db.sqlalchemy import api as db_api
 from dcmanager.rpc import client as rpc_client
 from dcmanager.tests.unit.api import test_root_controller as testroot
 from dcmanager.tests.unit.api.v1.controllers.mixins import APIMixin
 from dcmanager.tests.unit.api.v1.controllers.mixins import PostMixin
-from dcmanager.tests.unit.common.subcloud import FAKE_SUBCLOUD_INSTALL_VALUES
+from dcmanager.tests.unit.common import fake_subcloud
 from dcmanager.tests import utils
 
 SAMPLE_SUBCLOUD_NAME = 'SubcloudX'
 SAMPLE_SUBCLOUD_DESCRIPTION = 'A Subcloud of mystery'
 
-FAKE_TENANT = utils.UUID1
+FAKE_ID = fake_subcloud.FAKE_ID
-FAKE_ID = '1'
+FAKE_URL = fake_subcloud.FAKE_URL
-FAKE_URL = '/v1.0/subclouds'
+WRONG_URL = fake_subcloud.WRONG_URL
-WRONG_URL = '/v1.0/wrong'
+FAKE_HEADERS = fake_subcloud.FAKE_HEADERS
-FAKE_HEADERS = {'X-Tenant-Id': FAKE_TENANT, 'X_ROLE': 'admin',
+FAKE_SUBCLOUD_DATA = fake_subcloud.FAKE_SUBCLOUD_DATA
-                'X-Identity-Status': 'Confirmed'}
+FAKE_BOOTSTRAP_VALUE = fake_subcloud.FAKE_BOOTSTRAP_VALUE
-
+FAKE_SUBCLOUD_INSTALL_VALUES = fake_subcloud.FAKE_SUBCLOUD_INSTALL_VALUES
-FAKE_SUBCLOUD_DATA = {"id": FAKE_ID,
-                      "name": "subcloud1",
-                      "description": "subcloud1 description",
-                      "location": "subcloud1 location",
-                      "system_mode": "duplex",
-                      "management_subnet": "192.168.101.0/24",
-                      "management_start_address": "192.168.101.2",
-                      "management_end_address": "192.168.101.50",
-                      "management_gateway_address": "192.168.101.1",
-                      "systemcontroller_gateway_address": "192.168.204.101",
-                      "deploy_status": consts.DEPLOY_STATE_DONE,
-                      "external_oam_subnet": "10.10.10.0/24",
-                      "external_oam_gateway_address": "10.10.10.1",
-                      "external_oam_floating_address": "10.10.10.12",
-                      "availability-status": "disabled"}
-
-FAKE_BOOTSTRAP_VALUE = {
-    'bootstrap-address': '10.10.10.12',
-    'sysadmin_password': base64.b64encode('testpass'.encode("utf-8"))
-}
 
 
 class Subcloud(object):
@@ -97,6 +79,8 @@ class Subcloud(object):
             data['systemcontroller_gateway_address']
         self.created_at = timeutils.utcnow()
         self.updated_at = timeutils.utcnow()
+        self.data_install = ''
+        self.data_upgrade = ''
 
 
 class FakeAddressPool(object):
@@ -1121,3 +1105,89 @@ class TestSubcloudAPIOther(testroot.DCManagerApiTest):
                               FAKE_ID + '/update_status',
                               headers=FAKE_HEADERS, params=data)
         mock_rpc_client().update_subcloud_endpoint_status.assert_not_called()
+
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    def test_get_config_file_path(self, mock_rpc_client):
+        sc = subclouds.SubcloudsController()
+        bootstrap_file = sc._get_config_file_path("subcloud1")
+        install_values = sc._get_config_file_path("subcloud1", "install_values")
+        deploy_config = sc._get_config_file_path("subcloud1", consts.DEPLOY_CONFIG)
+        self.assertEqual(bootstrap_file, "/opt/dc/ansible/subcloud1.yml")
+        self.assertEqual(install_values, "/opt/dc/ansible/subcloud1/install_values.yml")
+        self.assertEqual(deploy_config, "/opt/dc/ansible/subcloud1_deploy_config.yml")
+
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(keyring, 'get_password')
+    def test_get_subcloud_db_install_values(
+        self, mock_keyring, mock_rpc_client):
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        encoded_password = base64.b64encode(
+            'bmc_password'.encode("utf-8")).decode('utf-8')
+        bmc_password = {'bmc_password': encoded_password}
+        install_data.update(bmc_password)
+        test_subcloud = copy.copy(FAKE_SUBCLOUD_DATA)
+        subcloud_info = Subcloud(test_subcloud, False)
+        subcloud_info.data_install = json.dumps(install_data)
+
+        sc = subclouds.SubcloudsController()
+        actual_result = sc._get_subcloud_db_install_values(subcloud_info)
+        actual_result.update({
+            'admin_password': 'adminpass'
+        })
+        install_data.update({
+            'ansible_become_pass': consts.TEMP_SYSADMIN_PASSWORD,
+            'ansible_ssh_pass': consts.TEMP_SYSADMIN_PASSWORD,
+            'admin_password': 'adminpass'
+        })
+        self.assertEqual(
+            json.loads(json.dumps(install_data)),
+            json.loads(json.dumps(actual_result)))
+
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(keyring, 'get_password')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_get_subcloud_db_install_values_without_bmc_password(
+        self, mock_db_api, mock_keyring, mock_rpc_client):
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        test_subcloud = copy.copy(FAKE_SUBCLOUD_DATA)
+
+        subcloud_info = Subcloud(test_subcloud, False)
+        subcloud_info.data_install = json.dumps(install_data)
+        mock_db_api.subcloud_get.return_value = subcloud_info
+
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.patch_json, FAKE_URL + '/' +
+                              FAKE_ID + '/reinstall',
+                              headers=FAKE_HEADERS)
+
+    @mock.patch.object(cutils, 'get_vault_load_files')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    @mock.patch.object(subclouds.SubcloudsController, '_get_subcloud_db_install_values')
+    @mock.patch.object(subclouds.SubcloudsController, '_validate_install_values')
+    def test_reinstall_subcloud(
+        self, moc_validate_install_values, mock_get_subcloud_db_install_values,
+        mock_db_api, mock_rpc_client, mock_get_vault_load_files):
+
+        # Return a fake subcloud database object
+        fake_subcloud = Subcloud(FAKE_SUBCLOUD_DATA, False)
+        mock_db_api.subcloud_get.return_value = fake_subcloud
+
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        encoded_password = base64.b64encode(
+            'bmc_password'.encode("utf-8")).decode('utf-8')
+        bmc_password = {'bmc_password': encoded_password}
+        install_data.update(bmc_password)
+
+        mock_get_subcloud_db_install_values.return_value = install_data
+        mock_db_api.subcloud_db_model_to_dict.return_value = FAKE_SUBCLOUD_DATA
+        mock_rpc_client().reinstall_subcloud.return_value = True
+        mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path')
+        response = self.app.patch_json(
+            FAKE_URL + '/' + FAKE_ID + '/reinstall',
+            headers=FAKE_HEADERS)
+        mock_rpc_client().reinstall_subcloud.assert_called_once_with(
+            mock.ANY,
+            FAKE_ID,
+            mock.ANY)
+        self.assertEqual(response.status_int, 200)

distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py

@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import base64
+
+from dcmanager.common import consts
+from dcmanager.tests import utils
+
+FAKE_TENANT = utils.UUID1
+FAKE_ID = '1'
+FAKE_URL = '/v1.0/subclouds'
+WRONG_URL = '/v1.0/wrong'
+
+FAKE_HEADERS = {'X-Tenant-Id': FAKE_TENANT, 'X_ROLE': 'admin',
+                'X-Identity-Status': 'Confirmed'}
+
+FAKE_SUBCLOUD_DATA = {"id": FAKE_ID,
+                      "name": "subcloud1",
+                      "description": "subcloud1 description",
+                      "location": "subcloud1 location",
+                      "system_mode": "duplex",
+                      "management_subnet": "192.168.101.0/24",
+                      "management_start_address": "192.168.101.2",
+                      "management_end_address": "192.168.101.50",
+                      "management_gateway_address": "192.168.101.1",
+                      "systemcontroller_gateway_address": "192.168.204.101",
+                      "deploy_status": consts.DEPLOY_STATE_DONE,
+                      "external_oam_subnet": "10.10.10.0/24",
+                      "external_oam_gateway_address": "10.10.10.1",
+                      "external_oam_floating_address": "10.10.10.12",
+                      "availability-status": "disabled"}
+
+FAKE_BOOTSTRAP_VALUE = {
+    'bootstrap-address': '10.10.10.12',
+    'sysadmin_password': base64.b64encode('testpass'.encode("utf-8"))
+}
+
+FAKE_SUBCLOUD_INSTALL_VALUES = {
+    "image": "http://192.168.101.2:8080/iso/bootimage.iso",
+    "software_version": "12.34",
+    "bootstrap_interface": "eno1",
+    "bootstrap_address": "128.224.151.183",
+    "bootstrap_address_prefix": 23,
+    "bmc_address": "128.224.64.180",
+    "bmc_username": "root",
+    "nexthop_gateway": "128.224.150.1",
+    "network_address": "128.224.144.0",
+    "network_mask": "255.255.254.0",
+    "install_type": 3,
+    "console_type": "tty0",
+    "bootstrap_vlan": 128,
+    "rootfs_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0",
+    "boot_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0",
+    "rd.net.timeout.ipv6dad": 300,
+}

distributedcloud/dcmanager/tests/unit/common/subcloud.py

@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2020 Wind River Systems, Inc.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-FAKE_SUBCLOUD_INSTALL_VALUES = {
-    "image": "http://192.168.101.2:8080/iso/bootimage.iso",
-    "software_version": "12.34",
-    "bootstrap_interface": "eno1",
-    "bootstrap_address": "128.224.151.183",
-    "bootstrap_address_prefix": 23,
-    "bmc_address": "128.224.64.180",
-    "bmc_username": "root",
-    "nexthop_gateway": "128.224.150.1",
-    "network_address": "128.224.144.0",
-    "network_mask": "255.255.254.0",
-    "install_type": 3,
-    "console_type": "tty0",
-    "bootstrap_vlan": 128,
-    "rootfs_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0",
-    "boot_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0",
-    "rd.net.timeout.ipv6dad": 300,
-}

distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py

@@ -34,8 +34,10 @@ from dcmanager.common import utils as cutils
 from dcmanager.db.sqlalchemy import api as db_api
 from dcmanager.manager import subcloud_manager
 from dcmanager.tests import base
+from dcmanager.tests.unit.common import fake_subcloud
 from dcmanager.tests import utils
 from dcorch.common import consts as dcorch_consts
+from tsconfig.tsconfig import SW_VERSION
 
 
 class FakeDCOrchAPI(object):
@@ -45,6 +47,7 @@ class FakeDCOrchAPI(object):
         self.remove_subcloud_sync_endpoint_type = mock.MagicMock()
         self.del_subcloud = mock.MagicMock()
         self.add_subcloud = mock.MagicMock()
+        self.update_subcloud_version = mock.MagicMock()
 
 
 class FakeDCManagerNotifications(object):
@@ -886,3 +889,153 @@ class TestSubcloudManager(base.DCManagerTestCase):
                                 payload=fake_payload)
         mock_thread_start.assert_called_once()
         mock_prepare_for_deployment.assert_called_once()
+
+    def test_get_ansible_filename(self):
+        sm = subcloud_manager.SubcloudManager()
+        filename = sm._get_ansible_filename('subcloud1',
+                                            consts.INVENTORY_FILE_POSTFIX)
+        self.assertEqual(filename, '/opt/dc/ansible/subcloud1_inventory.yml')
+
+    def test_compose_install_command(self):
+        sm = subcloud_manager.SubcloudManager()
+        install_command = sm.compose_install_command(
+            'subcloud1', '/opt/dc/ansible/subcloud1_inventory.yml')
+        self.assertEqual(
+            install_command,
+            [
+                'ansible-playbook', subcloud_manager.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
+                '-i', '/opt/dc/ansible/subcloud1_inventory.yml', '--limit', 'subcloud1',
+                '-e', "@/opt/dc/ansible/subcloud1/install_values.yml"
+            ]
+        )
+
+    def test_compose_apply_command(self):
+        sm = subcloud_manager.SubcloudManager()
+        apply_command = sm.compose_apply_command(
+            'subcloud1', '/opt/dc/ansible/subcloud1_inventory.yml')
+        self.assertEqual(
+            apply_command,
+            [
+                'ansible-playbook', subcloud_manager.ANSIBLE_SUBCLOUD_PLAYBOOK, '-i',
+                '/opt/dc/ansible/subcloud1_inventory.yml', '--limit', 'subcloud1', '-e',
+                "override_files_dir='/opt/dc/ansible' region_name=subcloud1"
+            ]
+        )
+
+    def test_compose_deploy_command(self):
+        sm = subcloud_manager.SubcloudManager()
+        fake_payload = {"sysadmin_password": "testpass",
+                        "deploy_playbook": "test_playbook.yaml",
+                        "deploy_overrides": "test_overrides.yaml",
+                        "deploy_chart": "test_chart.yaml",
+                        "deploy_config": "subcloud1.yaml"}
+        deploy_command = sm.compose_deploy_command(
+            'subcloud1', '/opt/dc/ansible/subcloud1_inventory.yml', fake_payload)
+        self.assertEqual(
+            deploy_command,
+            [
+                'ansible-playbook', 'test_playbook.yaml', '-e',
+                '@/opt/dc/ansible/subcloud1_deploy_values.yml', '-i',
+                '/opt/dc/ansible/subcloud1_inventory.yml', '--limit', 'subcloud1'
+            ]
+        )
+
+    @mock.patch.object(
+        subcloud_manager.SubcloudManager, '_create_intermediate_ca_cert')
+    @mock.patch.object(
+        subcloud_manager.SubcloudManager, 'compose_install_command')
+    @mock.patch.object(
+        subcloud_manager.SubcloudManager, 'compose_apply_command')
+    @mock.patch.object(subcloud_manager, 'db_api')
+    @mock.patch.object(cutils, 'create_subcloud_inventory')
+    @mock.patch.object(threading.Thread, 'start')
+    @mock.patch.object(subcloud_manager, 'keyring')
+    def test_reinstall_subcloud_with_image(
+        self, mock_keyring, mock_thread_start,
+        mock_create_subcloud_inventory, mock_db_api,
+        mock_compose_apply_command, mock_compose_install_command,
+        mock_create_intermediate_ca_cert):
+
+        values = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0)
+        values['deploy_status'] = consts.DEPLOY_STATE_PRE_DEPLOY
+        fake_install_values = fake_subcloud.FAKE_SUBCLOUD_INSTALL_VALUES
+        fake_install_values['software_version'] = SW_VERSION
+        fake_payload = {
+            "bmc_password": "bmc_pass",
+            "install_values": fake_install_values}
+        fake_subcloud_result = Subcloud(values, False)
+
+        mock_db_api.subcloud_get.return_value = fake_subcloud_result
+        mock_db_api.subcloud_update.return_value = fake_subcloud_result
+        sm = subcloud_manager.SubcloudManager()
+        mock_keyring.get_password.return_value = "testpassword"
+
+        sm.reinstall_subcloud(self.ctx, values['id'], payload=fake_payload)
+        mock_keyring.get_password.assert_called_once()
+        mock_create_subcloud_inventory.assert_called_once()
+        mock_compose_install_command.assert_called_once()
+        mock_compose_apply_command.assert_called_once()
+        mock_thread_start.assert_called_once()
+
+    @mock.patch.object(
+        subcloud_manager.SubcloudManager, '_create_intermediate_ca_cert')
+    @mock.patch.object(cutils, "get_vault_load_files")
+    @mock.patch.object(
+        subcloud_manager.SubcloudManager, 'compose_install_command')
+    @mock.patch.object(
+        subcloud_manager.SubcloudManager, 'compose_apply_command')
+    @mock.patch.object(subcloud_manager, 'db_api')
+    @mock.patch.object(cutils, 'create_subcloud_inventory')
+    @mock.patch.object(threading.Thread, 'start')
+    @mock.patch.object(subcloud_manager, 'keyring')
+    def test_reinstall_subcloud_without_image(
+        self, mock_keyring, mock_thread_start, mock_create_subcloud_inventory,
+        mock_db_api, mock_compose_apply_command, mock_compose_install_command,
+        mock_get_vault_load_files, mock_create_intermediate_ca_cert):
+
+        values = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0)
+        values['deploy_status'] = consts.DEPLOY_STATE_PRE_DEPLOY
+        fake_install_values = fake_subcloud.FAKE_SUBCLOUD_INSTALL_VALUES
+        fake_install_values['software_version'] = SW_VERSION
+        del fake_install_values['image']
+        fake_payload = {
+            "bmc_password": "bmc_pass",
+            "install_values": fake_install_values}
+        fake_subcloud_result = Subcloud(values, False)
+
+        mock_db_api.subcloud_get.return_value = fake_subcloud_result
+        mock_db_api.subcloud_update.return_value = fake_subcloud_result
+        sm = subcloud_manager.SubcloudManager()
+        mock_keyring.get_password.return_value = "testpassword"
+        mock_get_vault_load_files.return_value = ("iso file path", "sig file path")
+
+        sm.reinstall_subcloud(self.ctx, values['id'], payload=fake_payload)
+        mock_keyring.get_password.assert_called_once()
+        mock_create_subcloud_inventory.assert_called_once()
+        mock_compose_install_command.assert_called_once()
+        mock_compose_apply_command.assert_called_once()
+        mock_thread_start.assert_called_once()
+
+    @mock.patch.object(subcloud_manager, 'db_api')
+    def test_reinstall_online_subcloud(self, mock_db_api):
+        data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0)
+        subcloud_result = Subcloud(data, True)
+        subcloud_result.availability_status = consts.AVAILABILITY_ONLINE
+        mock_db_api.subcloud_get.return_value = subcloud_result
+        sm = subcloud_manager.SubcloudManager()
+        self.assertRaises(exceptions.SubcloudNotOffline,
+                          sm.reinstall_subcloud, self.ctx,
+                          data['id'], data)
+
+    @mock.patch.object(subcloud_manager, 'db_api')
+    def test_reinstall_subcloud_software_not_match(self, mock_db_api):
+        fake_install_values = fake_subcloud.FAKE_SUBCLOUD_INSTALL_VALUES
+        data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0)
+        subcloud_result = Subcloud(data, True)
+        subcloud_result.availability_status = consts.AVAILABILITY_OFFLINE
+        mock_db_api.subcloud_get.return_value = subcloud_result
+        data.update({'install_values': fake_install_values})
+        sm = subcloud_manager.SubcloudManager()
+        self.assertRaises(exceptions.BadRequest,
+                          sm.reinstall_subcloud, self.ctx,
+                          data['id'], data)

distributedcloud/dcmanager/tests/unit/orchestrator/states/fakes.py

@@ -9,7 +9,7 @@ import uuid
 from dcmanager.common import consts
 from oslo_utils import timeutils
 
-from dcmanager.tests.unit.common.subcloud import FAKE_SUBCLOUD_INSTALL_VALUES
+from dcmanager.tests.unit.common.fake_subcloud import FAKE_SUBCLOUD_INSTALL_VALUES
 
 
 PREVIOUS_PREVIOUS_VERSION = '01.23'