starlingx/distcloud
Code Issues Proposed changes

Implement the state for Simplex Upgrade

Browse Source 
The Simplex Upgrade step upgrades the simplex subcloud to the load
imported in dc-vault corresponding to the target software release.

Required static data is captured at subcloud add and referenced for
the simplex upgrade.

Dynamic data is retrieved from the online subcloud; including the
bmc credentials which are obtained via a synchronized service within
project allowing for access to the bmc credentials.

BMC password retrieval endpoint_cache integration
with https://review.opendev.org/#/c/736004/

Tests Performed:
- Subcloud upgrade (install) with load corresponding
  to System Controller
- Subcloud add persists install data
- Simplex upgrade persists dynamic upgrade data from start of
  upgrade simplex
- Obtain bmc password from online subcloud
- Obtain bmc password from install data, in case the
  bmc is not configured

Pending for subsequent commit:
- tox unit tests

Change-Id: Ie047139280a5780bfe47b9f9959f4c6781d6f3fa
Story: 2007403
Task: 40024
Signed-off-by: John Kung <john.kung@windriver.com>
This commit is contained in:
John Kung 2020-06-09 10:22:56 -04:00
parent 91e6c8a6d7
commit 1012dd2896
21 changed files with 631 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
distributedcloud/__init__.py Normal file
View File

2
distributedcloud/dccommon/consts.py
View File

@ -33,6 +33,7 @@ CLOUD_0 = "RegionOne"
VIRTUAL_MASTER_CLOUD = "SystemController"
SW_UPDATE_DEFAULT_TITLE = "all clouds default"
LOAD_VAULT_DIR = '/opt/dc-vault/loads'
USER_HEADER_VALUE = "distcloud"
USER_HEADER = {'User-Header': USER_HEADER_VALUE}
@ -41,3 +42,4 @@ ADMIN_USER_NAME = "admin"
ADMIN_PROJECT_NAME = "admin"
SYSINV_USER_NAME = "sysinv"
DCMANAGER_USER_NAME = "dcmanager"
SERVICES_USER_NAME = "services"

75
distributedcloud/dccommon/drivers/openstack/barbican.py Normal file
View File

@ -0,0 +1,75 @@
# Copyright 2016 Ericsson AB
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement.
#
from barbicanclient import client
from oslo_log import log
from dccommon import consts as dccommon_consts
from dccommon.drivers import base
from dccommon import exceptions
LOG = log.getLogger(__name__)
API_VERSION = 'v1'
class BarbicanClient(base.DriverBase):
"""Barbican driver.
The session needs to be associated with synchronized 'services' project
in order for the client to get the host bmc password.
"""
def __init__(
self, region, session, endpoint_type=dccommon_consts.KS_ENDPOINT_DEFAULT):
try:
self.barbican_client = client.Client(
API_VERSION,
session=session,
region_name=region,
interface=endpoint_type)
self.region_name = region
except exceptions.ServiceUnavailable:
raise
def get_host_bmc_password(self, host_uuid):
"""Get the Board Management Controller password corresponding to the host
:param host_uuid The host uuid
"""
secrets = self.barbican_client.secrets.list()
for secret in secrets:
if secret.name == host_uuid:
secret_ref = secret.secret_ref
break
else:
return
secret = self.barbican_client.secrets.get(secret_ref)
bmc_password = secret.payload
return bmc_password

7
distributedcloud/dccommon/drivers/openstack/sdk_platform.py
View File

@ -23,6 +23,7 @@ from oslo_log import log
from oslo_utils import timeutils
from dccommon import consts
from dccommon.drivers.openstack.barbican import BarbicanClient
from dccommon.drivers.openstack.fm import FmClient
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
@ -39,6 +40,7 @@ STALE_TOKEN_DURATION_STEP = 20
KEYSTONE_CLIENT_NAME = 'keystone'
SYSINV_CLIENT_NAME = 'sysinv'
FM_CLIENT_NAME = 'fm'
BARBICAN_CLIENT_NAME = 'barbican'
LOG = log.getLogger(__name__)
@ -46,13 +48,15 @@ LOCK_NAME = 'dc-openstackdriver-platform'
SUPPORTED_REGION_CLIENTS = [
SYSINV_CLIENT_NAME,
FM_CLIENT_NAME
FM_CLIENT_NAME,
BARBICAN_CLIENT_NAME,
]
# region client type and class mappings
region_client_class_map = {
SYSINV_CLIENT_NAME: SysinvClient,
FM_CLIENT_NAME: FmClient,
BARBICAN_CLIENT_NAME: BarbicanClient,
}
@ -68,6 +72,7 @@ class OpenStackDriver(object):
self.keystone_client = None
self.sysinv_client = None
self.fm_client = None
self.barbican_client = None
if region_clients:
# check if the requested clients are in the supported client list

33
distributedcloud/dccommon/drivers/openstack/sysinv_v1.py
View File

@ -130,6 +130,39 @@ class SysinvClient(base.DriverBase):
action_value = 'unlock'
return self._do_host_action(host_id, action_value)
def configure_bmc_host(self,
host_id,
bm_username,
bm_ip,
bm_password,
bm_type='ipmi'):
"""Configure bmc of a host"""
patch = [
{'op': 'replace',
'path': '/bm_username',
'value': bm_username},
{'op': 'replace',
'path': '/bm_ip',
'value': bm_ip},
{'op': 'replace',
'path': '/bm_password',
'value': bm_password},
{'op': 'replace',
'path': '/bm_type',
'value': bm_type},
]
return self.sysinv_client.ihost.update(host_id, patch)
def power_on_host(self, host_id):
"""Power on a host"""
action_value = 'power-on'
return self._do_host_action(host_id, action_value)
def power_off_host(self, host_id):
"""Power off a host"""
action_value = 'power-off'
return self._do_host_action(host_id, action_value)
def get_management_interface(self, hostname):
"""Get the management interface for a host."""
interfaces = self.sysinv_client.iinterface.list(hostname)

3
distributedcloud/dcmanager/common/install_consts.py → distributedcloud/dccommon/install_consts.py
View File

@ -31,3 +31,6 @@ MANDATORY_INSTALL_VALUES = [
'bmc_password',
'install_type'
]
ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/install.yml'

18
distributedcloud/dcmanager/manager/subcloud_install.py → distributedcloud/dccommon/subcloud_install.py
View File

@ -24,19 +24,17 @@ from eventlet.green import subprocess
import json
import netaddr
import os
import socket
from oslo_log import log as logging
from six.moves.urllib import error as urllib_error
from six.moves.urllib import parse
from six.moves.urllib import request
import socket
from dccommon import consts
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
from dcmanager.common import consts
from dcmanager.common import exceptions
from dcmanager.common import install_consts
from oslo_log import log as logging
from dccommon import exceptions
from dccommon import install_consts
LOG = logging.getLogger(__name__)
@ -88,7 +86,7 @@ class SubcloudInstall(object):
ks_client = KeystoneClient()
session = ks_client.endpoint_cache.get_session_from_token(
context.auth_token, context.project)
self.sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session)
self.sysinv_client = SysinvClient(consts.CLOUD_0, session)
self.name = subcloud_name
self.input_iso = None
self.www_root = None
@ -288,14 +286,14 @@ class SubcloudInstall(object):
msg = "Error: Downloading file %s may be interrupted: %s" % (
values['image'], e)
LOG.error(msg)
raise exceptions.DCManagerException(
raise exceptions.DCCommonException(
resource=self.name,
msg=msg)
except Exception as e:
msg = "Error: Could not download file %s: %s" % (
values['image'], e)
LOG.error(msg)
raise exceptions.DCManagerException(
raise exceptions.DCCommonException(
resource=self.name,
msg=msg)

10
distributedcloud/dcmanager/api/controllers/v1/subclouds.py
View File

@ -21,6 +21,7 @@
from requests_toolbelt.multipart import decoder
import base64
import json
import keyring
from netaddr import AddrFormatError
from netaddr import IPAddress
@ -39,6 +40,7 @@ from pecan import request
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
from dccommon import exceptions as dccommon_exceptions
from dccommon import install_consts
from keystoneauth1 import exceptions as keystone_exceptions
@ -48,7 +50,6 @@ from dcmanager.api.controllers import restcomm
from dcmanager.common import consts
from dcmanager.common import exceptions
from dcmanager.common.i18n import _
from dcmanager.common import install_consts
from dcmanager.common import utils
from dcmanager.db import api as db_api
@ -495,6 +496,10 @@ class SubcloudsController(object):
# if group_id has been omitted from payload, use 'Default'.
group_id = payload.get('group_id',
consts.DEFAULT_SUBCLOUD_GROUP_ID)
data_install = None
if 'install_values' in payload:
data_install = json.dumps(payload['install_values'])
subcloud = db_api.subcloud_create(
context,
payload['name'],
@ -508,7 +513,8 @@ class SubcloudsController(object):
payload['systemcontroller_gateway_address'],
consts.DEPLOY_STATE_NONE,
False,
group_id)
group_id,
data_install=data_install)
return subcloud
@index.when(method='GET', template='json')

10
distributedcloud/dcmanager/common/consts.py
View File

@ -123,6 +123,7 @@ DEPLOY_STATE_PRE_INSTALL = 'pre-install'
DEPLOY_STATE_PRE_INSTALL_FAILED = 'pre-install-failed'
DEPLOY_STATE_INSTALLING = 'installing'
DEPLOY_STATE_INSTALL_FAILED = 'install-failed'
DEPLOY_STATE_INSTALLED = 'installed'
DEPLOY_STATE_BOOTSTRAPPING = 'bootstrapping'
DEPLOY_STATE_BOOTSTRAP_FAILED = 'bootstrap-failed'
DEPLOY_STATE_DEPLOYING = 'deploying'
@ -147,3 +148,12 @@ DEPLOY_COMMON_FILE_OPTIONS = [
DEPLOY_OVERRIDES,
DEPLOY_CHART
]
DC_LOG_DIR = '/var/log/dcmanager/'
INVENTORY_FILE_POSTFIX = '_inventory.yml'
# The following password is just a temporary and internal password that is used
# after a remote install as part of the upgrade. The real sysadmin password
# will be restored af the subcloud is re-managed at the end of the upgrade.
TEMP_SYSADMIN_PASSWORD = 'St8rlingX*'

27
distributedcloud/dcmanager/common/utils.py
View File

@ -207,3 +207,30 @@ def get_filename_by_prefix(dir_path, prefix):
if filename.startswith(prefix):
return filename
return None
def create_subcloud_inventory(subcloud, inventory_file):
"""Create the ansible inventory file for the specified subcloud"""
# Delete the file if it already exists
delete_subcloud_inventory(inventory_file)
with open(inventory_file, 'w') as f_out_inventory:
f_out_inventory.write(
'---\n'
'all:\n'
' vars:\n'
' ansible_ssh_user: sysadmin\n'
' hosts:\n'
' ' + subcloud['name'] + ':\n'
' ansible_host: ' +
subcloud['bootstrap-address'] + '\n'
)
def delete_subcloud_inventory(inventory_file):
"""Delete the ansible inventory file for the specified subcloud"""
# Delete the file if it exists
if os.path.isfile(inventory_file):
os.remove(inventory_file)

11
distributedcloud/dcmanager/db/api.py
View File

@ -66,6 +66,8 @@ def subcloud_db_model_to_dict(subcloud):
"openstack-installed": subcloud.openstack_installed,
"systemcontroller-gateway-ip":
subcloud.systemcontroller_gateway_ip,
"data_install": subcloud.data_install,
"data_upgrade": subcloud.data_upgrade,
"created-at": subcloud.created_at,
"updated-at": subcloud.updated_at,
"group_id": subcloud.group_id}
@ -76,14 +78,14 @@ def subcloud_create(context, name, description, location, software_version,
management_subnet, management_gateway_ip,
management_start_ip, management_end_ip,
systemcontroller_gateway_ip, deploy_status,
openstack_installed, group_id):
openstack_installed, group_id, data_install=None):
"""Create a subcloud."""
return IMPL.subcloud_create(context, name, description, location,
software_version,
management_subnet, management_gateway_ip,
management_start_ip, management_end_ip,
systemcontroller_gateway_ip, deploy_status,
openstack_installed, group_id)
openstack_installed, group_id, data_install)
def subcloud_get(context, subcloud_id):
@ -115,12 +117,13 @@ def subcloud_update(context, subcloud_id, management_state=None,
availability_status=None, software_version=None,
description=None, location=None, audit_fail_count=None,
deploy_status=None, openstack_installed=None,
group_id=None):
group_id=None, data_install=None, data_upgrade=None):
"""Update a subcloud or raise if it does not exist."""
return IMPL.subcloud_update(context, subcloud_id, management_state,
availability_status, software_version,
description, location, audit_fail_count,
deploy_status, openstack_installed, group_id)
deploy_status, openstack_installed, group_id,
data_install, data_upgrade)
def subcloud_destroy(context, subcloud_id):

15
distributedcloud/dcmanager/db/sqlalchemy/api.py
View File

@ -213,7 +213,7 @@ def subcloud_create(context, name, description, location, software_version,
management_subnet, management_gateway_ip,
management_start_ip, management_end_ip,
systemcontroller_gateway_ip, deploy_status,
openstack_installed, group_id):
openstack_installed, group_id, data_install=None):
with write_session() as session:
subcloud_ref = models.Subcloud()
subcloud_ref.name = name
@ -231,6 +231,8 @@ def subcloud_create(context, name, description, location, software_version,
subcloud_ref.audit_fail_count = 0
subcloud_ref.openstack_installed = openstack_installed
subcloud_ref.group_id = group_id
if data_install is not None:
subcloud_ref.data_install = data_install
session.add(subcloud_ref)
return subcloud_ref
@ -239,8 +241,11 @@ def subcloud_create(context, name, description, location, software_version,
def subcloud_update(context, subcloud_id, management_state=None,
availability_status=None, software_version=None,
description=None, location=None, audit_fail_count=None,
deploy_status=None, openstack_installed=None,
group_id=None):
deploy_status=None,
openstack_installed=None,
group_id=None,
data_install=None,
data_upgrade=None):
with write_session() as session:
subcloud_ref = subcloud_get(context, subcloud_id)
if management_state is not None:
@ -255,8 +260,12 @@ def subcloud_update(context, subcloud_id, management_state=None,
subcloud_ref.location = location
if audit_fail_count is not None:
subcloud_ref.audit_fail_count = audit_fail_count
if data_install is not None:
subcloud_ref.data_install = data_install
if deploy_status is not None:
subcloud_ref.deploy_status = deploy_status
if data_upgrade is not None:
subcloud_ref.data_upgrade = data_upgrade
if openstack_installed is not None:
subcloud_ref.openstack_installed = openstack_installed
if group_id is not None:

40
distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/007_add_subcloud_install.py Normal file
View File

@ -0,0 +1,40 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement.
#
from sqlalchemy import Column
from sqlalchemy import MetaData
from sqlalchemy import Table
from sqlalchemy import Text
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
subclouds = Table('subclouds', meta, autoload=True)
# Add the 'data_install' to persist data_install data
subclouds.create_column(Column('data_install', Text))
# Add the data_upgrade which persist over an upgrade
subclouds.create_column(Column('data_upgrade', Text))
def downgrade(migrate_engine):
raise NotImplementedError('Database downgrade is unsupported.')

3
distributedcloud/dcmanager/db/sqlalchemy/models.py
View File

@ -99,7 +99,9 @@ class Subcloud(BASE, DCManagerBase):
software_version = Column(String(255))
management_state = Column(String(255))
availability_status = Column(String(255))
data_install = Column(String())
deploy_status = Column(String(255))
data_upgrade = Column(String())
management_subnet = Column(String(255))
management_gateway_ip = Column(String(255))
management_start_ip = Column(String(255), unique=True)
@ -107,6 +109,7 @@ class Subcloud(BASE, DCManagerBase):
openstack_installed = Column(Boolean, nullable=False, default=False)
systemcontroller_gateway_ip = Column(String(255))
audit_fail_count = Column(Integer)
# multiple subclouds can be in a particular group
group_id = Column(Integer,
ForeignKey('subcloud_group.id'))

17
distributedcloud/dcmanager/manager/states/base.py
View File

@ -8,9 +8,11 @@ import six
from oslo_log import log as logging
from dccommon.drivers.openstack.barbican import BarbicanClient
from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
from dcmanager.common import consts
from dcmanager.common import context
LOG = logging.getLogger(__name__)
@ -20,6 +22,7 @@ class BaseState(object):
def __init__(self):
super(BaseState, self).__init__()
self.context = context.get_admin_context()
def debug_log(self, strategy_step, details):
LOG.debug("Stage: %s, State: %s, Subcloud: %s, Details: %s"
@ -35,6 +38,13 @@ class BaseState(object):
self.get_region_name(strategy_step),
details))
def error_log(self, strategy_step, details):
LOG.error("Stage: %s, State: %s, Subcloud: %s, Details: %s"
% (strategy_step.stage,
strategy_step.state,
self.get_region_name(strategy_step),
details))
@staticmethod
def get_region_name(strategy_step):
"""Get the region name for a strategy step"""
@ -64,6 +74,13 @@ class BaseState(object):
"""
return SysinvClient(region_name, session)
@staticmethod
def get_barbican_client(region_name, session):
"""construct a barbican client
"""
return BarbicanClient(region_name, session)
@abc.abstractmethod
def perform_state_action(self, strategy_step):
"""Perform the action for this state on the strategy_step"""

363
distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py
View File

@ -3,9 +3,22 @@
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
import json
import keyring
import os
from dccommon.install_consts import ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK
from dccommon.subcloud_install import SubcloudInstall
from dcmanager.common import consts
from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
from dcmanager.common import utils
from dcmanager.db import api as db_api
from dcmanager.manager.states.base import BaseState
from dcmanager.manager.states.upgrade import utils as upgrade_utils
from oslo_log import log as logging
from tsconfig.tsconfig import SW_VERSION
LOG = logging.getLogger(__name__)
@ -22,9 +35,349 @@ class UpgradingSimplexState(BaseState):
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
LOG.warning("UpgradingSimplexState has not been implemented yet.")
# When we return from this method without throwing an exception, the
# state machine can proceed to the next state
LOG.warning("Faking transition to next state")
LOG.info("Performing simplex upgrade for subcloud %s" %
strategy_step.subcloud.name)
subcloud_sysinv_client = None
subcloud_barbican_client = None
try:
subcloud_ks_client = self.get_keystone_client(strategy_step.subcloud.name)
subcloud_sysinv_client = self.get_sysinv_client(
strategy_step.subcloud.name,
subcloud_ks_client.session)
subcloud_barbican_client = self.get_barbican_client(
strategy_step.subcloud.name,
subcloud_ks_client.session)
except Exception:
# if getting the token times out, the orchestrator may have
# restarted and subcloud may be offline; so will attempt
# to use the persisted values
message = ("Simplex upgrade perform_subcloud_install "
"subcloud %s failed to get subcloud client" %
strategy_step.subcloud.name)
self.error_log(strategy_step, message)
pass
# Check whether subcloud is already re-installed with N+1 load
target_version = SW_VERSION
if self._check_load_already_active(
target_version, subcloud_sysinv_client):
self.info_log(strategy_step,
"Load:%s already active" % target_version)
return True
# Check whether subcloud supports redfish, and if not, fail.
# This needs to be inferred from absence of install_values as
# there is currrently no external api to query.
install_values = self.get_subcloud_upgrade_install_values(
strategy_step, subcloud_sysinv_client, subcloud_barbican_client)
local_ks_client = self.get_keystone_client()
# Upgrade the subcloud to the install_values image
self.perform_subcloud_install(
strategy_step, local_ks_client.session, install_values)
def _check_load_already_active(self, target_version, subcloud_sysinv_client):
"""Check if the target_version is already active in subcloud"""
if subcloud_sysinv_client:
current_loads = subcloud_sysinv_client.get_loads()
for load in current_loads:
if (load.software_version == target_version and
load.state == 'active'):
return True
return False
def get_subcloud_upgrade_install_values(
self, strategy_step,
subcloud_sysinv_client, subcloud_barbican_client):
"""Get the data required for the remote subcloud install.
subcloud data_install are obtained from:
dcmanager database:
subcloud.subcloud_install_initial::for values which are persisted at subcloud_add time
INSTALL: (needed for upgrade install)
bootstrap_interface
bootstrap_vlan
bootstrap_address
bootstrap_address_prefix
install_type # could also be from host-show
# This option can be set to extend the installing stage timeout value
# wait_for_timeout: 3600
# Set this options for https with self-signed certificate
# no_check_certificate
# Override default filesystem device: also from host-show, but is static.
# rootfs_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0"
# boot_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0"
BOOTSTRAP: (also needed for bootstrap)
# If the subcloud's bootstrap IP interface and the system controller are not on the
# same network then the customer must configure a default route or static route
# so that the Central Cloud can login bootstrap the newly installed subcloud.
# If nexthop_gateway is specified and the network_address is not specified then a
# default route will be configured. Otherwise, if a network_address is specified
then
# a static route will be configured.
nexthop_gateway: default_route_address
network_address: static_route_address
network_mask: static_route_mask
subcloud.data_upgrade - persist for upgrade duration
for values from subcloud online sysinv host-show (persist since upgrade-start)
bmc_address # sysinv_v1 host-show
bmc_username # sysinv_v1 host-show
for values from barbican_client (as barbican user), or from upgrade-start:
bmc_password --- obtain from barbican_client as barbican user
"""
install_values = {'name': strategy_step.subcloud.name}
install_values.update(
self._get_subcloud_upgrade_load_info(strategy_step))
upgrade_data_install_values = self._get_subcloud_upgrade_data_install(
strategy_step)
install_values.update(upgrade_data_install_values)
install_values.update(
self._get_subcloud_upgrade_data(
strategy_step, subcloud_sysinv_client, subcloud_barbican_client))
# Check bmc values
if not self._bmc_data_available(install_values):
if self._bmc_data_available(upgrade_data_install_values):
# It is possible the bmc data is only latched on install if it
# was not part of the deployment configuration
install_values.update({
'bmc_address':
upgrade_data_install_values.get('bmc_address'),
'bmc_username':
upgrade_data_install_values.get('bmc_username'),
'bmc_password':
upgrade_data_install_values.get('bmc_password'),
})
else:
message = ("Failed to get bmc credentials for subcloud %s" %
strategy_step.subcloud.name)
raise Exception(message)
self.info_log(strategy_step,
"get_subcloud_upgrade_data_install %s" % install_values)
return install_values
@staticmethod
def _bmc_data_available(bmc_values):
if (not bmc_values.get('bmc_username') or
not bmc_values.get('bmc_address') or
not bmc_values.get('bmc_password')):
return False
return True
def _get_subcloud_upgrade_load_info(self, strategy_step):
"""Get the subcloud upgrade load information"""
# The 'software_version' is the active running load on SystemController
matching_iso, _ = upgrade_utils.get_vault_load_files(SW_VERSION)
if not os.path.isfile(matching_iso):
message = ("Failed to get upgrade load info for subcloud %s" %
strategy_step.subcloud.name)
raise Exception(message)
load_info = {'software_version': SW_VERSION,
'image': matching_iso}
return load_info
def _get_subcloud_upgrade_data_install(self, strategy_step):
"""Get subcloud upgrade data_install from persisted values"""
upgrade_data_install = {}
subcloud = db_api.subcloud_get(self.context, strategy_step.subcloud_id)
if not subcloud.data_install:
message = ("Failed to get upgrade data from install "
"for subcloud %s." %
strategy_step.subcloud.name)
LOG.warn(message)
raise Exception(message)
data_install = json.loads(subcloud.data_install)
# base64 encoded sysadmin_password is default
upgrade_data_install.update({
'ansible_become_pass': consts.TEMP_SYSADMIN_PASSWORD,
'ansible_ssh_pass': consts.TEMP_SYSADMIN_PASSWORD,
})
# Get mandatory bootstrap info from data_install
# bootstrap_address is referenced in SubcloudInstall
# bootstrap-address is referenced in create_subcloud_inventory and
# subcloud manager.
# todo(jkung): refactor to just use one bootstrap address index
upgrade_data_install.update({
'bootstrap_interface': data_install.get('bootstrap_interface'),
'bootstrap-address': data_install.get('bootstrap_address'),
'bootstrap_address': data_install.get('bootstrap_address'),
'bootstrap_address_prefix': data_install.get('bootstrap_address_prefix'),
'bmc_username': data_install.get('bmc_username'),
'bmc_address': data_install.get('bmc_address'),
'bmc_password': data_install.get('bmc_password'),
})
# optional bootstrap parameters
optional_bootstrap_parameters = [
'nexthop_gateway', # default route address
'network_address', # static route address
'network_mask', # static route mask
'bootstrap_vlan',
'wait_for_timeout',
'no_check_certificate',
]
for p in optional_bootstrap_parameters:
if p in data_install:
upgrade_data_install.update({p: data_install.get(p)})
return upgrade_data_install
def _get_subcloud_upgrade_data(
self, strategy_step, subcloud_sysinv_client, subcloud_barbican_client):
"""Get the subcloud data required for upgrades.
In case the subcloud is no longer reachable, get upgrade_data from
persisted database values. For example, this may be required in
the scenario where the subcloud experiences an unexpected error
(e.g. loss of power) and this step needs to be rerun.
"""
volatile_data_install = {}
if subcloud_sysinv_client is None:
# subcloud is not reachable, use previously saved values
subcloud = db_api.subcloud_get(
self.context, strategy_step.subcloud_id)
if subcloud.data_upgrade:
return json.loads(subcloud.data_upgrade)
else:
message = ('Cannot retrieve upgrade data install '
'for subcloud: %s' %
strategy_step.subcloud.name)
raise Exception(message)
subcloud_system = subcloud_sysinv_client.get_system()
if subcloud_system.system_type != 'All-in-one':
message = ('subcloud %s install unsupported for system type: %s' %
(strategy_step.subcloud.name,
subcloud_system.system_type))
raise Exception(message)
host = subcloud_sysinv_client.get_host('controller-0')
install_type = self._get_install_type(host)
bmc_password = None
if subcloud_barbican_client:
bmc_password = subcloud_barbican_client.get_host_bmc_password(host.uuid)
volatile_data_install.update({
'bmc_address': host.bm_ip,
'bmc_username': host.bm_username,
'bmc_password': bmc_password,
'install_type': install_type,
'boot_device': host.boot_device,
'rootfs_device': host.rootfs_device,
})
# Persist the volatile data
db_api.subcloud_update(
self.context, strategy_step.subcloud_id,
data_upgrade=json.dumps(volatile_data_install))
admin_password = str(keyring.get_password('CGCS', 'admin'))
volatile_data_install.update({'admin_password': admin_password})
return volatile_data_install
@staticmethod
def _get_install_type(host):
if 'lowlatency' in host.subfunctions.split(','):
lowlatency = True
else:
lowlatency = False
if 'graphical' in host.console.split(','): # graphical console
if lowlatency:
install_type = 5
else:
install_type = 3
else: # serial console
if lowlatency:
install_type = 4
else:
install_type = 2
return install_type
def perform_subcloud_install(self, strategy_step, session, install_values):
db_api.subcloud_update(
self.context, strategy_step.subcloud_id,
deploy_status=consts.DEPLOY_STATE_PRE_INSTALL)
self.context.auth_token = session.get_token()
self.context.project = session.get_project_id()
try:
install = SubcloudInstall(
self.context, strategy_step.subcloud.name)
install.prep(consts.ANSIBLE_OVERRIDES_PATH,
install_values)
except Exception as e:
db_api.subcloud_update(
self.context, strategy_step.subcloud_id,
deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED)
self.error_log(strategy_step, e.message)
# TODO(jkung): cleanup to be implemented within SubcloudInstall
install.cleanup()
raise
ansible_subcloud_inventory_file = os.path.join(
consts.ANSIBLE_OVERRIDES_PATH,
strategy_step.subcloud.name + INVENTORY_FILE_POSTFIX)
# Create the ansible inventory for the upgrade subcloud
utils.create_subcloud_inventory(install_values,
ansible_subcloud_inventory_file)
# SubcloudInstall.prep creates data_install.yml (install overrides)
install_command = [
"ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
"-i", ansible_subcloud_inventory_file,
"-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
strategy_step.subcloud.name + '/' + "install_values.yml"
]
# Run the remote install playbook
db_api.subcloud_update(
self.context, strategy_step.subcloud_id,
deploy_status=consts.DEPLOY_STATE_INSTALLING)
try:
install.install(consts.DC_LOG_DIR, install_command)
except Exception as e:
db_api.subcloud_update(
self.context, strategy_step.subcloud_id,
deploy_status=consts.DEPLOY_STATE_INSTALL_FAILED)
self.error_log(strategy_step, e.message)
install.cleanup()
raise
db_api.subcloud_update(
self.context, strategy_step.subcloud_id,
deploy_status=consts.DEPLOY_STATE_INSTALLED)
install.cleanup()
LOG.info("Successfully installed subcloud %s" %
strategy_step.subcloud.name)

54
distributedcloud/dcmanager/manager/subcloud_manager.py
View File

@ -39,11 +39,13 @@ from dccommon import consts as dccommon_consts
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
from dccommon import kubeoperator
from dccommon.subcloud_install import SubcloudInstall
from dcorch.common import consts as dcorch_consts
from dcorch.rpc import client as dcorch_rpc_client
from dcmanager.common import consts
from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
from dcmanager.common import context
from dcmanager.common import exceptions
from dcmanager.common.i18n import _
@ -51,7 +53,6 @@ from dcmanager.common import manager
from dcmanager.common import utils
from dcmanager.db import api as db_api
from dcmanager.manager.subcloud_install import SubcloudInstall
from fm_api import constants as fm_const
from fm_api import fm_api
@ -63,12 +64,10 @@ LOG = logging.getLogger(__name__)
ADDN_HOSTS_DC = 'dnsmasq.addn_hosts_dc'
# Subcloud configuration paths
INVENTORY_FILE_POSTFIX = '_inventory.yml'
ANSIBLE_SUBCLOUD_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/bootstrap.yml'
ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/install.yml'
DC_LOG_DIR = '/var/log/dcmanager/'
USERS_TO_REPLICATE = [
'sysinv',
@ -79,8 +78,6 @@ USERS_TO_REPLICATE = [
'barbican',
'dcmanager']
SERVICES_USER = 'services'
SC_INTERMEDIATE_CERT_DURATION = "87600h"
SC_INTERMEDIATE_CERT_RENEW_BEFORE = "720h"
CERT_NAMESPACE = "dc-cert"
@ -191,7 +188,6 @@ class SubcloudManager(manager.Manager):
"""Add subcloud and notify orchestrators.
:param context: request context object
:param name: name of subcloud to add
:param payload: subcloud configuration
"""
LOG.info("Adding subcloud %s." % payload['name'])
@ -303,7 +299,8 @@ class SubcloudManager(manager.Manager):
dccommon_consts.ADMIN_USER_NAME)
admin_project = m_ks_client.get_project_by_name(
dccommon_consts.ADMIN_PROJECT_NAME)
services_project = m_ks_client.get_project_by_name(SERVICES_USER)
services_project = m_ks_client.get_project_by_name(
dccommon_consts.SERVICES_USER_NAME)
sysinv_user = m_ks_client.get_user_by_name(
dccommon_consts.SYSINV_USER_NAME)
dcmanager_user = m_ks_client.get_user_by_name(
@ -342,14 +339,14 @@ class SubcloudManager(manager.Manager):
]
del payload['sysadmin_password']
payload['users'] = dict()
for user in USERS_TO_REPLICATE:
payload['users'][user] = \
str(keyring.get_password(user, SERVICES_USER))
str(keyring.get_password(
user, dccommon_consts.SERVICES_USER_NAME))
# Create the ansible inventory for the new subcloud
self._create_subcloud_inventory(payload,
utils.create_subcloud_inventory(payload,
ansible_subcloud_inventory_file)
# create subcloud intermediate certificate and pass in keys
@ -467,7 +464,7 @@ class SubcloudManager(manager.Manager):
context, subcloud.id,
deploy_status=consts.DEPLOY_STATE_INSTALLING)
try:
install.install(DC_LOG_DIR, install_command)
install.install(consts.DC_LOG_DIR, install_command)
except Exception as e:
db_api.subcloud_update(
context, subcloud.id,
@ -490,7 +487,7 @@ class SubcloudManager(manager.Manager):
# Run the ansible boostrap-subcloud playbook
log_file = \
DC_LOG_DIR + subcloud.name + '_bootstrap_' + \
consts.DC_LOG_DIR + subcloud.name + '_bootstrap_' + \
str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) \
+ '.log'
with open(log_file, "w") as f_out_log:
@ -519,7 +516,7 @@ class SubcloudManager(manager.Manager):
context, subcloud.id,
deploy_status=consts.DEPLOY_STATE_DEPLOYING)
log_file = \
DC_LOG_DIR + subcloud.name + '_deploy_' + \
consts.DC_LOG_DIR + subcloud.name + '_deploy_' + \
str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) \
+ '.log'
with open(log_file, "w") as f_out_log:
@ -569,35 +566,6 @@ class SubcloudManager(manager.Manager):
# restart dnsmasq so it can re-read our addn_hosts file.
os.system("pkill -HUP dnsmasq")
def _create_subcloud_inventory(self,
subcloud,
inventory_file):
"""Create the inventory file for the specified subcloud"""
# Delete the file if it already exists
if os.path.isfile(inventory_file):
os.remove(inventory_file)
with open(inventory_file, 'w') as f_out_inventory:
f_out_inventory.write(
'---\n'
'all:\n'
' vars:\n'
' ansible_ssh_user: sysadmin\n'
' hosts:\n'
' ' + subcloud['name'] + ':\n'
' ansible_host: ' +
subcloud['bootstrap-address'] + '\n'
)
def _delete_subcloud_inventory(self,
inventory_file):
"""Delete the inventory file for the specified subcloud"""
# Delete the file if it exists
if os.path.isfile(inventory_file):
os.remove(inventory_file)
def _write_subcloud_ansible_config(self, context, payload):
"""Create the override file for usage with the specified subcloud"""
@ -736,7 +704,7 @@ class SubcloudManager(manager.Manager):
raise e
# Delete the ansible inventory for the new subcloud
self._delete_subcloud_inventory(ansible_subcloud_inventory_file)
utils.delete_subcloud_inventory(ansible_subcloud_inventory_file)
# Delete the subcloud intermediate certificate
SubcloudManager._delete_subcloud_cert(subcloud.name)

4
distributedcloud/dcmanager/tests/base.py
View File

@ -20,6 +20,7 @@
# of an applicable Wind River license agreement.
#
import json
import sqlalchemy
from oslo_config import cfg
@ -65,7 +66,8 @@ SUBCLOUD_SAMPLE_DATA_0 = [
"10.10.10.12", # external_oam_floating_address
"testpass", # sysadmin_password
1, # group_id
consts.DEPLOY_STATE_DONE # deploy_status
consts.DEPLOY_STATE_DONE, # deploy_status
json.dumps({'data_install': 'test data install values'}), # data_install
]

9
distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py
View File

@ -30,6 +30,7 @@ import threading
from dccommon import consts as dccommon_consts
from dcmanager.common import consts
from dcmanager.common import exceptions
from dcmanager.common import utils as cutils
from dcmanager.db.sqlalchemy import api as db_api
from dcmanager.manager import subcloud_manager
from dcmanager.tests import base
@ -123,6 +124,7 @@ class Subcloud(object):
data['external_oam_floating_address']
self.systemcontroller_gateway_ip = \
data['systemcontroller_gateway_address']
self.data_install = data['data_install']
self.created_at = timeutils.utcnow()
self.updated_at = timeutils.utcnow()
@ -159,6 +161,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
'deploy_status': "not-deployed",
'openstack_installed': False,
'group_id': 1,
'data_install': 'data from install',
}
values.update(kwargs)
return db_api.subcloud_create(ctxt, **values)
@ -172,15 +175,13 @@ class TestSubcloudManager(base.DCManagerTestCase):
@mock.patch.object(subcloud_manager.SubcloudManager,
'_create_intermediate_ca_cert')
@mock.patch.object(subcloud_manager.SubcloudManager,
'_delete_subcloud_inventory')
@mock.patch.object(cutils, 'delete_subcloud_inventory')
@mock.patch.object(subcloud_manager, 'KeystoneClient')
@mock.patch.object(subcloud_manager, 'db_api')
@mock.patch.object(subcloud_manager, 'SysinvClient')
@mock.patch.object(subcloud_manager.SubcloudManager,
'_create_addn_hosts_dc')
@mock.patch.object(subcloud_manager.SubcloudManager,
'_create_subcloud_inventory')
@mock.patch.object(cutils, 'create_subcloud_inventory')
@mock.patch.object(subcloud_manager.SubcloudManager,
'_write_subcloud_ansible_config')
@mock.patch.object(subcloud_manager,

3
distributedcloud/dcmanager/tests/utils.py
View File

@ -119,4 +119,5 @@ def create_subcloud_dict(data_list):
'external_oam_floating_address': data_list[21],
'sysadmin_password': data_list[22],
'group_id': data_list[23],
'deploy_status': data_list[24]}
'deploy_status': data_list[24],
'data_install': data_list[25]}

1
distributedcloud/requirements.txt
View File

@ -41,6 +41,7 @@ oslo.utils>=3.20.0 # Apache-2.0
oslo.versionedobjects>=1.17.0 # Apache-2.0
sqlalchemy-migrate>=0.11.0 # Apache-2.0
python-openstackclient!=3.10.0,>=3.3.0 # Apache-2.0
python-barbicanclient>=4.5.2
python-neutronclient>=6.3.0 # Apache-2.0
python-cinderclient>=2.1.0 # Apache-2.0
python-novaclient>=7.1.0 # Apache-2.0