Remote install of sub-cloud controller-0

This update extends the remote sub-cloud deployment to include
the installation of controller-0, which provides a complete ZTP
solution. This is an optional capability that leverages
Redfish Virtual Media Controller(rvmc).

Optional install-value parameters, are added to the dcmanager
subcloud add command, which provides the data required by the
rvmc tool, and update-iso.sh script.

Once install-values are provided, the dcmanager prepares for the
installation and performs the following:
. Downloads an iso image from the url provided, and creates a new
  bootable image based on the install values.
  The new image contains essential info to config a bootstrap ip
  interface that is used to reach the system controller
. Creates a config file for the rvmc tool
. Creates an ansible override file which is used by the install
  playbook

In the next step, the dcmanager runs the install playbook to
install the controller-0 of the subcloud. Once the installation
is completed, the bootstrapping of the sub-cloud would continue.

Story: 2006980
Task: 37715

Depends-On: https://review.opendev.org/#/c/702786/
Change-Id: Id3a1b97adb83a0da51cd54253bdb77e1d729327e
Signed-off-by: Tao Liu <tao.liu@windriver.com>

distributedcloud/dcmanager/api/controllers/v1/subclouds.py

@@ -13,12 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-# Copyright (c) 2017-2019 Wind River Systems, Inc.
+# Copyright (c) 2017-2020 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
 
 import keyring
+from netaddr import AddrFormatError
 from netaddr import IPAddress
 from netaddr import IPNetwork
 from netaddr import IPRange
@@ -41,6 +42,7 @@ from dcmanager.api.controllers import restcomm
 from dcmanager.common import consts
 from dcmanager.common import exceptions
 from dcmanager.common.i18n import _
+from dcmanager.common import install_consts
 from dcmanager.db import api as db_api
 from dcmanager.drivers.openstack.sysinv_v1 import SysinvClient
 from dcmanager.rpc import client as rpc_client
@@ -222,6 +224,73 @@ class SubcloudsController(object):
             LOG.exception(e)
             pecan.abort(400, _("oam_floating_address invalid: %s") % e)
 
+    @staticmethod
+    def _validate_install_values(payload):
+        install_values = payload.get('install_values')
+        bmc_password = payload.get('bmc_password')
+        if not bmc_password:
+            pecan.abort(400, _('subcloud bmc_password required'))
+        payload['install_values'].update({'bmc_password': bmc_password})
+
+        for k in install_consts.MANDATORY_INSTALL_VALUES:
+            if k not in install_values:
+                pecan.abort(400, _('Mandatory install value %s not present')
+                            % k)
+
+        if (install_values['install_type'] not in
+                range(install_consts.SUPPORTED_INSTALL_TYPES)):
+            pecan.abort(400, _("install_type invalid: %s") %
+                        install_values['install_type'])
+
+        try:
+            ip_version = (IPAddress(install_values['bootstrap_address']).
+                          version)
+        except AddrFormatError as e:
+            LOG.exception(e)
+            pecan.abort(400, _("bootstrap_address invalid: %s") % e)
+
+        try:
+            bmc_address = IPAddress(install_values['bmc_address'])
+        except AddrFormatError as e:
+            LOG.exception(e)
+            pecan.abort(400, _("bmc_address invalid: %s") % e)
+
+        if bmc_address.version != ip_version:
+            pecan.abort(400, _("bmc_address and bootstrap_address "
+                               "must be the same IP version"))
+
+        if 'nexthop_gateway' in install_values:
+            try:
+                gateway_ip = IPAddress(install_values['nexthop_gateway'])
+            except AddrFormatError:
+                LOG.exception(e)
+                pecan.abort(400, _("nexthop_gateway address invalid: %s") % e)
+            if gateway_ip.version != ip_version:
+                pecan.abort(400, _("nexthop_gateway and bootstrap_address "
+                                   "must be the same IP version"))
+
+        if ('network_address' in install_values and
+                'nexthop_gateway' not in install_values):
+            pecan.abort(400, _("nexthop_gateway is required when "
+                               "network_address is present"))
+
+        if 'nexthop_gateway' and 'network_address' in install_values:
+            if 'network_mask' not in install_values:
+                pecan.abort(400, _("The network mask is required when network "
+                                   "address is present"))
+
+            network_str = (install_values['network_address'] + '/' +
+                           str(install_values['network_mask']))
+            try:
+                network = validate_network_str(network_str, 1)
+            except ValidateFail as e:
+                LOG.exception(e)
+                pecan.abort(400, _("network address invalid: %s") % e)
+
+            if network.version != ip_version:
+                pecan.abort(400, _("network address and bootstrap address "
+                                   "must be the same IP version"))
+
     def _get_subcloud_users(self):
         """Get the subcloud users and passwords from keyring"""
         DEFAULT_SERVICE_PROJECT_NAME = 'services'
@@ -425,10 +494,10 @@ class SubcloudsController(object):
             if not external_oam_floating_ip:
                 pecan.abort(400, _('external_oam_floating_address required'))
 
-            subcloud_password = \
-                payload.get('subcloud_password')
-            if not subcloud_password:
-                pecan.abort(400, _('subcloud_password required'))
+            sysadmin_password = \
+                payload.get('sysadmin_password')
+            if not sysadmin_password:
+                pecan.abort(400, _('subcloud sysadmin_password required'))
 
             self._validate_subcloud_config(context,
                                            name,
@@ -441,6 +510,9 @@ class SubcloudsController(object):
                                            external_oam_floating_ip,
                                            systemcontroller_gateway_ip)
 
+            if 'install_values' in payload:
+                self._validate_install_values(payload)
+
             try:
                 # Ask dcmanager-manager to add the subcloud.
                 # It will do all the real work...

distributedcloud/dcmanager/common/consts.py

@@ -91,6 +91,10 @@ SW_UPDATE_DEFAULT_TITLE = "all clouds default"
 
 # Subcloud deploy status states
 DEPLOY_STATE_NONE = 'not-deployed'
+DEPLOY_STATE_PRE_INSTALL = 'pre-install'
+DEPLOY_STATE_PRE_INSTALL_FAILED = 'pre-install-failed'
+DEPLOY_STATE_INSTALLING = 'installing'
+DEPLOY_STATE_INSTALL_FAILED = 'install-failed'
 DEPLOY_STATE_BOOTSTRAPPING = 'bootstrapping'
 DEPLOY_STATE_BOOTSTRAP_FAILED = 'bootstrap-failed'
 DEPLOY_STATE_DEPLOYING = 'deploying'

distributedcloud/dcmanager/common/install_consts.py

@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+SUPPORTED_INSTALL_TYPES = 6
+
+MANDATORY_INSTALL_VALUES = [
+    'image',
+    'software_version',
+    'bootstrap_interface',
+    'bootstrap_address',
+    'bootstrap_address_prefix',
+    'bmc_address',
+    'bmc_username',
+    'bmc_password',
+    'install_type'
+]

distributedcloud/dcmanager/drivers/openstack/sysinv_v1.py

@@ -143,3 +143,25 @@ class SysinvClient(base.DriverBase):
 
         # Get a list of containerized applications the system knows of
         return self.sysinv_client.app.list()
+
+    def get_system(self):
+        """Get the system."""
+        systems = self.sysinv_client.isystem.list()
+        return systems[0]
+
+    def get_service_parameters(self, name, value):
+        """Get service parameters for a given name."""
+        opts = []
+        opt = dict()
+        opt['field'] = name
+        opt['value'] = value
+        opt['op'] = 'eq'
+        opt['type'] = ''
+        opts.append(opt)
+        parameters = self.sysinv_client.service_parameter.list(q=opts)
+        return parameters
+
+    def get_registry_image_tags(self, image_name):
+        """Get the image tags for an image from the local registry"""
+        image_tags = self.sysinv_client.registry_image.tags(image_name)
+        return image_tags

distributedcloud/dcmanager/manager/subcloud_audit_manager.py

@@ -93,6 +93,13 @@ class SubcloudAuditManager(manager.Manager):
                 break
 
         for subcloud in db_api.subcloud_get_all(self.context):
+            if (subcloud.deploy_status not in
+                    [consts.DEPLOY_STATE_DONE,
+                     consts.DEPLOY_STATE_DEPLOYING,
+                     consts.DEPLOY_STATE_DEPLOY_FAILED]):
+                LOG.info("Skip subcloud %s audit, deploy_status: %s" %
+                         (subcloud.name, subcloud.deploy_status))
+                continue
             # Create a new greenthread for each subcloud to allow the audits
             # to be done in parallel. If there are not enough greenthreads
             # in the pool, this will block until one becomes available.

distributedcloud/dcmanager/manager/subcloud_install.py

@@ -0,0 +1,397 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+import base64
+import datetime
+import json
+import netaddr
+import os
+import socket
+import subprocess
+
+from six.moves.urllib import error as urllib_error
+from six.moves.urllib import parse
+from six.moves.urllib import request
+
+from dcmanager.common import consts
+from dcmanager.common import exceptions
+from dcmanager.common import install_consts
+from dcmanager.drivers.openstack.sysinv_v1 import SysinvClient
+from dcorch.drivers.openstack.keystone_v3 import KeystoneClient
+
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+BOOT_MENU_TIMEOUT = '5'
+RVMC_NAME_PREFIX = 'rvmc'
+RVMC_IMAGE_NAME = 'docker.io/starlingx/rvmc'
+SUBCLOUD_ISO_PATH = '/opt/platform/iso'
+SUBCLOUD_ISO_DOWNLOAD_PATH = '/www/pages/iso'
+UPDATE_ISO_COMMAND = '/usr/local/bin/update-iso.sh'
+NETWORK_SCRIPTS = '/etc/sysconfig/network-scripts'
+NETWORK_INTERFACE_PREFIX = 'ifcfg'
+NETWORK_ROUTE_PREFIX = 'route'
+
+
+OPTIONAL_INSTALL_VALUES = [
+    'nexthop_gateway',
+    'network_address',
+    'network_mask',
+    'console_type',
+    'bootstrap_vlan',
+    'rootfs_device',
+    'boot_device'
+]
+
+UPDATE_ISO_OPTIONS = {
+    'install_type': '-d',
+    'console_type': '-p',
+    'rootfs_device': '-p',
+    'boot_device': '-p'
+}
+
+BMC_OPTIONS = {
+    'bmc_address',
+    'bmc_username',
+    'bmc_password',
+}
+
+
+class SubcloudInstall(object):
+    """Class to encapsulate the subcloud install operations"""
+
+    def __init__(self, context, subcloud_name):
+        ks_client = KeystoneClient()
+        session = ks_client.endpoint_cache.get_session_from_token(
+            context.auth_token, context.project)
+        self.sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session)
+        self.name = subcloud_name
+        self.input_iso = None
+        self.output_iso = None
+
+    @staticmethod
+    def config_device(ks_cfg, interface, vlan=False):
+        device_cfg = "%s/%s-%s" % (NETWORK_SCRIPTS, NETWORK_INTERFACE_PREFIX,
+                                   interface)
+        ks_cfg.write("\tcat << EOF > " + device_cfg + "\n")
+        ks_cfg.write("DEVICE=" + interface + "\n")
+        ks_cfg.write("BOOTPROTO=none\n")
+        ks_cfg.write("ONBOOT=yes\n")
+        if vlan:
+            ks_cfg.write("VLAN=yes\n")
+
+    @staticmethod
+    def config_ip_address(ks_cfg, values):
+        ks_cfg.write("IPADDR=" + values['bootstrap_address'] + "\n")
+        ks_cfg.write(
+            "PREFIX=" + str(values['bootstrap_address_prefix']) + "\n")
+
+    @staticmethod
+    def config_default_route(ks_cfg, values, ip_version):
+        if ip_version == 4:
+            ks_cfg.write("DEFROUTE=yes\n")
+            ks_cfg.write("GATEWAY=" + values['nexthop_gateway'] + "\n")
+        else:
+            ks_cfg.write("IPV6INIT=yes\n")
+            ks_cfg.write("IPV6_DEFROUTE=yes\n")
+            ks_cfg.write("IPV6_DEFAULTGW=" + values['nexthop_gateway'] + "\n")
+
+    @staticmethod
+    def config_static_route(ks_cfg, interface, values, ip_version):
+        if ip_version == 4:
+            route_cfg = "%s/%s-%s" % (NETWORK_SCRIPTS, NETWORK_ROUTE_PREFIX,
+                                      interface)
+            ks_cfg.write("\tcat << EOF > " + route_cfg + "\n")
+            ks_cfg.write("ADDRESS0=" + values['network_address'] + "\n")
+            ks_cfg.write("NETMASK0=" + str(values['network_mask']) + "\n")
+            ks_cfg.write("GATEWAY0=" + values['nexthop_gateway'] + "\n")
+        else:
+            route_cfg = "%s/%s6-%s" % (NETWORK_SCRIPTS, NETWORK_ROUTE_PREFIX,
+                                       interface)
+            ks_cfg.write("\tcat << EOF > " + route_cfg + "\n")
+            route_args = "%s/%s via %s dev %s\n" % (values['network_address'],
+                                                    values['network_mask'],
+                                                    values['nexthop_gateway'],
+                                                    interface)
+            ks_cfg.write(route_args)
+        ks_cfg.write("EOF\n\n")
+
+    def get_oam_address(self):
+        oam_pool = self.sysinv_client.get_oam_address_pool()
+        try:
+            oam_address = netaddr.IPAddress(oam_pool.floating_address)
+            if oam_address.version == 6:
+                return "[%s]" % oam_address
+            else:
+                return str(oam_address)
+        except netaddr.AddrFormatError:
+            return oam_pool.floating_address
+
+    def get_image_base_url(self):
+        system = self.sysinv_client.get_system()
+
+        # get the protocol
+        https_enabled = system.capabilities.get('https_enabled', False)
+        protocol = 'https' if https_enabled else 'http'
+
+        # get the configured http or https port
+        value = 'https_port' if https_enabled else 'http_port'
+        http_parameters = self.sysinv_client.get_service_parameters('name',
+                                                                    value)
+        port = getattr(http_parameters[0], 'value')
+
+        return "%s://%s:%s" % (protocol, self.get_oam_address(), port)
+
+    def get_image_tag(self, image_name):
+        tags = self.sysinv_client.get_registry_image_tags(image_name)
+        if not tags:
+            msg = ("Error: Image % not found in the local registry." %
+                   image_name)
+            LOG.error(msg)
+            raise exceptions.NotFound()
+        tag = getattr(tags[0], 'tag')
+        return tag
+
+    @staticmethod
+    def create_rvmc_config_file(override_path, payload):
+
+        LOG.debug("create rvmc config file")
+        rvmc_config_file = os.path.join(override_path, 'rvmc-config.yaml')
+
+        with open(rvmc_config_file, 'w') as f_out_rvmc_config_file:
+            for k, v in payload.items():
+                if k in BMC_OPTIONS or k == 'image':
+                    f_out_rvmc_config_file.write(k + ': ' + v + '\n')
+
+    def create_install_override_file(self, override_path, payload):
+
+        LOG.debug("create install override file")
+        rvmc_image = RVMC_IMAGE_NAME + ':' + self.get_image_tag(
+            RVMC_IMAGE_NAME)
+        install_override_file = os.path.join(override_path,
+                                             'install_values.yml')
+        rvmc_name = "%s-%s" % (RVMC_NAME_PREFIX, self.name)
+        host_name = socket.gethostname()
+
+        with open(install_override_file, 'w') as f_out_override_file:
+            f_out_override_file.write(
+                '---'
+                '\npassword_change: true'
+                '\nrvmc_image: ' + rvmc_image +
+                '\nrvmc_name: ' + rvmc_name +
+                '\nhost_name: ' + host_name +
+                '\nrvmc_config_dir: ' + override_path
+                + '\n'
+            )
+            for k, v in payload.items():
+                f_out_override_file.write("%s: %s\n" % (k, json.dumps(v)))
+
+    def create_ks_conf_file(self, filename, values):
+        try:
+            with open(filename, 'w') as f:
+                # create ks-addon.cfg
+                default_route = False
+                static_route = False
+                if 'nexthop_gateway' in values:
+                    if 'network_address' in values:
+                        static_route = True
+                    else:
+                        default_route = True
+
+                f.write("OAM_DEV=" + str(values['bootstrap_interface']) + "\n")
+
+                vlan_id = None
+                if 'bootstrap_vlan' in values:
+                    vlan_id = values['bootstrap_vlan']
+                    f.write("OAM_VLAN=" + str(vlan_id) + "\n\n")
+
+                interface = "$OAM_DEV"
+                self.config_device(f, interface)
+
+                ip_version = netaddr.IPAddress(
+                    values['bootstrap_address']).version
+                if vlan_id is None:
+                    self.config_ip_address(f, values)
+                    if default_route:
+                        self.config_default_route(f, values, ip_version)
+                f.write("EOF\n\n")
+
+                route_interface = interface
+                if vlan_id is not None:
+                    vlan_interface = "$OAM_DEV.$OAM_VLAN"
+                    self.config_device(f, vlan_interface, vlan=True)
+                    self.config_ip_address(f, values)
+                    if default_route:
+                        self.config_default_route(f, values, ip_version)
+                    f.write("EOF\n")
+                    route_interface = vlan_interface
+
+                if static_route:
+                    self.config_static_route(f, route_interface,
+                                             values, ip_version)
+        except IOError as e:
+            LOG.error("Failed to open file: %s", filename)
+            LOG.exception(e)
+            raise e
+
+    def update_iso(self, override_path, values):
+        output_iso_dir = os.path.join(SUBCLOUD_ISO_PATH,
+                                      str(values['software_version']))
+        if not os.path.isdir(output_iso_dir):
+            os.mkdir(output_iso_dir, 0o755)
+
+        try:
+            if parse.urlparse(values['image']).scheme:
+                url = values['image']
+            else:
+                path = os.path.abspath(values['image'])
+                url = parse.urljoin('file:', request.pathname2url(path))
+            filename = os.path.join(override_path, 'bootimage.iso')
+            LOG.info("Downloading %s to %s", url, override_path)
+            self.input_iso, _ = request.urlretrieve(url, filename)
+            LOG.info("Downloaded %s to %s", url, self.input_iso)
+        except urllib_error.ContentTooShortError as e:
+            msg = "Error: Downloading file %s may be interrupted: %s" % (
+                values['image'], e)
+            LOG.error(msg)
+            raise exceptions.DCManagerException(
+                resource=self.name,
+                msg=msg)
+        except Exception as e:
+            msg = "Error: Could not download file %s: %s" % (
+                values['image'], e)
+            LOG.error(msg)
+            raise exceptions.DCManagerException(
+                resource=self.name,
+                msg=msg)
+
+        output_iso_file = os.path.join(output_iso_dir,
+                                       self.name + 'bootimage.iso')
+        if os.path.exists(output_iso_file):
+            os.remove(output_iso_file)
+
+        update_iso_cmd = [
+            UPDATE_ISO_COMMAND,
+            "-i", self.input_iso,
+            "-o", output_iso_file
+        ]
+        for k in UPDATE_ISO_OPTIONS.keys():
+            if k in values:
+                if k == 'install_type':
+                    update_iso_cmd += [UPDATE_ISO_OPTIONS[k],
+                                       str(values[k])]
+                else:
+                    update_iso_cmd += [UPDATE_ISO_OPTIONS[k],
+                                       (k + '=' + values[k])]
+
+        # create ks-addon.cfg
+        addon_cfg = os.path.join(override_path, 'ks-addon.cfg')
+        self.create_ks_conf_file(addon_cfg, values)
+
+        update_iso_cmd += ['-t', BOOT_MENU_TIMEOUT]
+        update_iso_cmd += ['-a', addon_cfg]
+
+        str_cmd = ' '.join(update_iso_cmd)
+        LOG.debug("update_iso_cmd:(%s)", str_cmd)
+        try:
+            with open(os.devnull, "w") as fnull:
+                subprocess.check_call(update_iso_cmd, stdout=fnull,
+                                      stderr=fnull)
+        except subprocess.CalledProcessError as ex:
+            msg = "Failed to update iso %s, " % str(update_iso_cmd)
+            LOG.error(msg)
+            raise ex
+        return output_iso_file
+
+    def cleanup(self):
+        if (self.input_iso is not None and
+                os.path.exists(self.input_iso)):
+            os.remove(self.input_iso)
+        if (self.output_iso is not None and
+                os.path.exists(self.output_iso)):
+            os.remove(self.output_iso)
+
+    def prep(self, override_path, payload):
+        """Update the iso image and create the config files for the subcloud"""
+        LOG.info("Prepare for %s remote install" % (self.name))
+        iso_values = {}
+        for k in install_consts.MANDATORY_INSTALL_VALUES:
+            if k in UPDATE_ISO_OPTIONS.keys():
+                iso_values[k] = payload.get(k)
+            if k not in BMC_OPTIONS:
+                iso_values[k] = payload.get(k)
+
+        for k in OPTIONAL_INSTALL_VALUES:
+            if k in payload:
+                iso_values[k] = payload.get(k)
+
+        override_path = os.path.join(override_path, self.name)
+        if not os.path.isdir(override_path):
+            os.mkdir(override_path, 0o755)
+
+        # update the default iso image based on the install values
+        self.output_iso = self.update_iso(override_path, iso_values)
+        software_version = str(payload['software_version'])
+
+        # remove the iso values from the payload
+        for k in iso_values:
+            if k in payload:
+                del payload[k]
+
+        # get the boot image url for bmc
+        payload['image'] = os.path.join(self.get_image_base_url(), 'iso',
+                                        software_version,
+                                        os.path.basename(self.output_iso))
+        # encode the bmc_password
+        encoded_password = base64.b64encode(
+            payload['bmc_password'].encode("utf-8"))
+        payload['bmc_password'] = str(encoded_password)
+
+        # create the rvmc config file
+        self.create_rvmc_config_file(override_path, payload)
+
+        # remove the bmc values from the payload
+        for k in BMC_OPTIONS:
+            if k in payload:
+                del payload[k]
+
+        # remove the boot image url from the payload
+        if 'image' in payload:
+            del payload['image']
+
+        # create the install override file
+        self.create_install_override_file(override_path, payload)
+
+    def install(self, log_file_dir, install_command):
+        log_file = (log_file_dir + self.name + '_install_' +
+                    str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S'))
+                    + '.log')
+        with open(log_file, "w") as f_out_log:
+            try:
+                subprocess.check_call(install_command,
+                                      stdout=f_out_log,
+                                      stderr=f_out_log)
+            except subprocess.CalledProcessError as e:
+                msg = ("Failed to install the subcloud %s, check individual "
+                       "log at %s for detailed output."
+                       % (self.name, log_file))
+                LOG.error(msg)
+                raise e

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Copyright (c) 2017-2019 Wind River Systems, Inc.
+# Copyright (c) 2017-2020 Wind River Systems, Inc.
 #
 # The right to copy, distribute, modify, or otherwise make use
 # of this software may be licensed only pursuant to the terms
@@ -46,6 +46,7 @@ from dcmanager.common.i18n import _
 from dcmanager.common import manager
 from dcmanager.db import api as db_api
 from dcmanager.drivers.openstack.sysinv_v1 import SysinvClient
+from dcmanager.manager.subcloud_install import SubcloudInstall
 
 from fm_api import constants as fm_const
 from fm_api import fm_api
@@ -61,7 +62,8 @@ ANSIBLE_OVERRIDES_PATH = '/opt/dc/ansible'
 ANSIBLE_SUBCLOUD_INVENTORY_FILE = 'subclouds.yml'
 ANSIBLE_SUBCLOUD_PLAYBOOK = \
     '/usr/share/ansible/stx-ansible/playbooks/bootstrap.yml'
-
+ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \
+    '/usr/share/ansible/stx-ansible/playbooks/install.yml'
 DC_LOG_DIR = '/var/log/dcmanager/'
 
 USERS_TO_REPLICATE = [
@@ -215,20 +217,24 @@ class SubcloudManager(manager.Manager):
 
             # Add the admin and service user passwords to the payload so they
             # get copied to the override file
-            payload['ansible_become_pass'] = payload['subcloud_password']
-            payload['ansible_ssh_pass'] = payload['subcloud_password']
+            payload['ansible_become_pass'] = payload['sysadmin_password']
+            payload['ansible_ssh_pass'] = payload['sysadmin_password']
             payload['admin_password'] = str(keyring.get_password('CGCS',
                                                                  'admin'))
 
+            if "install_values" in payload:
+                payload['install_values']['ansible_ssh_pass'] = \
+                    payload['sysadmin_password']
+
             if "deploy_playbook" in payload:
                 payload['deploy_values']['ansible_become_pass'] = \
-                    payload['subcloud_password']
+                    payload['sysadmin_password']
                 payload['deploy_values']['ansible_ssh_pass'] = \
-                    payload['subcloud_password']
+                    payload['sysadmin_password']
                 payload['deploy_values']['admin_password'] = \
                     str(keyring.get_password('CGCS', 'admin'))
 
-            del payload['subcloud_password']
+            del payload['sysadmin_password']
 
             payload['users'] = dict()
             for user in USERS_TO_REPLICATE:
@@ -244,14 +250,16 @@ class SubcloudManager(manager.Manager):
             if "deploy_playbook" in payload:
                 self._write_deploy_files(payload)
 
-            # Update the subcloud to bootstrapping
-            try:
-                db_api.subcloud_update(
-                    context, subcloud.id,
-                    deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPING)
-            except Exception as e:
-                LOG.exception(e)
-                raise e
+            install_command = None
+            if "install_values" in payload:
+                install_command = [
+                    "ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
+                    "-i", ANSIBLE_OVERRIDES_PATH + '/' +
+                    ANSIBLE_SUBCLOUD_INVENTORY_FILE,
+                    "--limit", subcloud.name,
+                    "-e", "@%s" % ANSIBLE_OVERRIDES_PATH + "/" +
+                          payload['name'] + '/' + "install_values.yml"
+                ]
 
             apply_command = [
                 "ansible-playbook", ANSIBLE_SUBCLOUD_PLAYBOOK, "-i",
@@ -281,7 +289,8 @@ class SubcloudManager(manager.Manager):
 
             apply_thread = threading.Thread(
                 target=self.run_deploy,
-                args=(apply_command, deploy_command, subcloud, context))
+                args=(install_command, apply_command, deploy_command, subcloud,
+                      payload, context))
             apply_thread.start()
 
             return db_api.subcloud_db_model_to_dict(subcloud)
@@ -295,7 +304,50 @@ class SubcloudManager(manager.Manager):
             raise e
 
     @staticmethod
-    def run_deploy(apply_command, deploy_command, subcloud, context):
+    def run_deploy(install_command, apply_command, deploy_command, subcloud,
+                   payload, context):
+
+        if install_command:
+            db_api.subcloud_update(
+                context, subcloud.id,
+                deploy_status=consts.DEPLOY_STATE_PRE_INSTALL)
+            try:
+                install = SubcloudInstall(context, subcloud.name)
+                install.prep(ANSIBLE_OVERRIDES_PATH, payload['install_values'])
+            except Exception as e:
+                LOG.exception(e)
+                db_api.subcloud_update(
+                    context, subcloud.id,
+                    deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED)
+                LOG.error(e.message)
+                install.cleanup()
+                return
+
+            # Run the remote install playbook
+            db_api.subcloud_update(
+                context, subcloud.id,
+                deploy_status=consts.DEPLOY_STATE_INSTALLING)
+            try:
+                install.install(DC_LOG_DIR, install_command)
+            except Exception as e:
+                db_api.subcloud_update(
+                    context, subcloud.id,
+                    deploy_status=consts.DEPLOY_STATE_INSTALL_FAILED)
+                LOG.error(e.message)
+                install.cleanup()
+                return
+            install.cleanup()
+            LOG.info("Successfully installed subcloud %s" % subcloud.name)
+
+        # Update the subcloud to bootstrapping
+        try:
+            db_api.subcloud_update(
+                context, subcloud.id,
+                deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPING)
+        except Exception as e:
+            LOG.exception(e)
+            raise e
+
         # Run the ansible boostrap-subcloud playbook
         log_file = \
             DC_LOG_DIR + subcloud.name + '_bootstrap_' + \
@@ -434,7 +486,8 @@ class SubcloudManager(manager.Manager):
             )
 
             for k, v in payload.items():
-                if k not in ['deploy_playbook', 'deploy_values']:
+                if k not in ['deploy_playbook', 'deploy_values',
+                             'install_values']:
                     f_out_overrides_file.write("%s: %s\n" % (k, json.dumps(v)))
 
     def _write_deploy_files(self, payload):

distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py

@@ -31,6 +31,7 @@ from dcmanager.rpc import client as rpc_client
 from dcmanager.tests.unit.api import test_root_controller as testroot
 from dcmanager.tests import utils
 
+
 FAKE_TENANT = utils.UUID1
 FAKE_ID = '1'
 FAKE_URL = '/v1.0/subclouds'
@@ -51,7 +52,25 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1",
                       "external_oam_gateway_address": "10.10.10.1",
                       "external_oam_floating_address": "10.10.10.12",
                       "availability-status": "disabled",
-                      "subcloud_password": "testpass"}
+                      "sysadmin_password": "testpass"}
+
+FAKE_SUBCLOUD_INSTALL_VALUES = {
+    "image": "http://192.168.101.2:8080/iso/bootimage.iso",
+    "software_version": "20.01",
+    "bootstrap_interface": "eno1",
+    "bootstrap_address": "128.224.151.183",
+    "bootstrap_address_prefix": 23,
+    "bmc_address": "128.224.64.180",
+    "bmc_username": "root",
+    "nexthop_gateway": "128.224.150.1",
+    "network_address": "128.224.144.0",
+    "network_mask": "255.255.254.0",
+    "install_type": 3,
+    "console_type": "tty0",
+    "bootstrap_vlan": 128,
+    "rootfs_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0",
+    "boot_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0"
+}
 
 
 class FakeAddressPool(object):
@@ -90,6 +109,27 @@ class TestSubclouds(testroot.DCManagerApiTest):
             data)
         self.assertEqual(response.status_int, 200)
 
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_with_install_values(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        data['bmc_password'] = 'bmc_password'
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        mock_rpc_client().add_subcloud.return_value = True
+        response = self.app.post_json(FAKE_URL,
+                                      headers=FAKE_HEADERS,
+                                      params=data)
+        self.assertEqual(response.status_int, 200)
+
     @mock.patch.object(subclouds.SubcloudsController,
                        '_get_management_address_pool')
     @mock.patch.object(rpc_client, 'ManagerClient')
@@ -178,6 +218,160 @@ class TestSubclouds(testroot.DCManagerApiTest):
                               self.app.post_json, WRONG_URL,
                               headers=FAKE_HEADERS, params=data)
 
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_no_bmc_password(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_missing_mandatory_values(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        del install_data['image']
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_invalid_type(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        install_data['install_type'] = 10
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_bad_bootstrap_ip(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        install_data['bootstrap_address'] = '192.168.1.256'
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_bad_bmc_ip(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        install_data['bmc_address'] = '128.224.64.280'
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_different_ip_version(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        install_data['nexthop_gateway'] = '192.168.1.2'
+        install_data['network_address'] = 'fd01:6::0'
+        install_data['bmc_address'] = 'fd01:6::7'
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_missing_network_gateway(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        del install_data['nexthop_gateway']
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
+    @mock.patch.object(subclouds.SubcloudsController,
+                       '_get_management_address_pool')
+    @mock.patch.object(rpc_client, 'ManagerClient')
+    @mock.patch.object(subclouds, 'db_api')
+    def test_post_subcloud_install_bad_network_address(
+            self, mock_db_api, mock_rpc_client,
+            mock_get_management_address_pool):
+        data = copy.copy(FAKE_SUBCLOUD_DATA)
+        install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES)
+        install_data['network_address'] = 'fd01:6::0'
+        install_data['network_mask'] = '63'
+        data.update({'install_values': install_data})
+        management_address_pool = FakeAddressPool('192.168.204.0', 24,
+                                                  '192.168.204.2',
+                                                  '192.168.204.100')
+        mock_get_management_address_pool.return_value = management_address_pool
+        six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
+                              self.app.post_json, FAKE_URL,
+                              headers=FAKE_HEADERS, params=data)
+
     @mock.patch.object(rpc_client, 'ManagerClient')
     def test_post_no_body(self, mock_rpc_client):
         data = {}

distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py

@@ -50,6 +50,22 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1",
                       "external_oam_subnet": "10.10.10.0/24",
                       "external_oam_gateway_address": "10.10.10.1",
                       "external_oam_floating_address": "10.10.10.12"}
+FAKE_SUBCLOUD_INSTALL_VALUES = {
+    'image': 'image: http://128.224.115.21/iso/bootimage.iso',
+    'software_version': '20.01',
+    'bootstrap_interface': 'enp0s3',
+    'bootstrap_address': '128.118.101.5',
+    'bootstrap_address_prefix': 23,
+    'bmc_address': '128.224.64.180',
+    'bmc_username': 'root',
+    'nexthop_gateway': '128.224.150.1',
+    'network_address': '128.224.144.0',
+    'network_mask': '255.255.254.0',
+    'install_type': 3,
+    'console_type': 'tty0',
+    'rootfs_device': '/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0',
+    'boot_device': ' /dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0'
+}
 
 
 class Controller(object):
@@ -152,7 +168,6 @@ class TestSubcloudManager(base.DCManagerTestCase):
         mock_create_addn_hosts.assert_called_once()
         mock_update_subcloud_inventory.assert_called_once()
         mock_write_subcloud_ansible_config.assert_called_once()
-        mock_db_api.subcloud_update.assert_called()
         mock_keyring.get_password.assert_called()
 
     @file_data(utils.get_data_filepath('dcmanager', 'subclouds'))

distributedcloud/dcmanager/tests/utils.py

@@ -131,7 +131,7 @@ def create_subcloud_dict(data_list):
             'external_oam_subnet': data_list[19],
             'external_oam_gateway_address': data_list[20],
             'external_oam_floating_address': data_list[21],
-            'subcloud_password': data_list[22]}
+            'sysadmin_password': data_list[22]}
 
 
 def create_route_dict(data_list):