2467 lines
110 KiB
Python
2467 lines
110 KiB
Python
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Task driver for completing node provisioning with Canonical MaaS 2.2+."""
|
|
|
|
import time
|
|
import logging
|
|
import re
|
|
import math
|
|
import yaml
|
|
|
|
from datetime import datetime
|
|
|
|
import drydock_provisioner.error as errors
|
|
import drydock_provisioner.config as config
|
|
import drydock_provisioner.objects.fields as hd_fields
|
|
import drydock_provisioner.objects.hostprofile as hostprofile
|
|
import drydock_provisioner.objects as objects
|
|
|
|
from drydock_provisioner.control.util import get_internal_api_href
|
|
from drydock_provisioner.orchestrator.actions.orchestrator import BaseAction
|
|
from drydock_provisioner.drivers.node.maasdriver.errors import RackControllerConflict
|
|
from drydock_provisioner.drivers.node.maasdriver.errors import ApiNotAvailable
|
|
|
|
import drydock_provisioner.drivers.node.maasdriver.models.fabric as maas_fabric
|
|
import drydock_provisioner.drivers.node.maasdriver.models.vlan as maas_vlan
|
|
import drydock_provisioner.drivers.node.maasdriver.models.subnet as maas_subnet
|
|
import drydock_provisioner.drivers.node.maasdriver.models.machine as maas_machine
|
|
import drydock_provisioner.drivers.node.maasdriver.models.tag as maas_tag
|
|
import drydock_provisioner.drivers.node.maasdriver.models.sshkey as maas_keys
|
|
import drydock_provisioner.drivers.node.maasdriver.models.boot_resource as maas_boot_res
|
|
import drydock_provisioner.drivers.node.maasdriver.models.rack_controller as maas_rack
|
|
import drydock_provisioner.drivers.node.maasdriver.models.partition as maas_partition
|
|
import drydock_provisioner.drivers.node.maasdriver.models.volumegroup as maas_vg
|
|
import drydock_provisioner.drivers.node.maasdriver.models.repository as maas_repo
|
|
import drydock_provisioner.drivers.node.maasdriver.models.domain as maas_domain
|
|
|
|
|
|
class BaseMaasAction(BaseAction):
|
|
def __init__(self, *args, maas_client=None):
|
|
super().__init__(*args)
|
|
|
|
self.maas_client = maas_client
|
|
|
|
self.logger = logging.getLogger(
|
|
config.config_mgr.conf.logging.nodedriver_logger_name)
|
|
|
|
def _add_detail_logs(self, node, machine, stage, result_type='all'):
|
|
result_details = machine.get_task_results(result_type=result_type)
|
|
for r in result_details:
|
|
if r.get_decoded_data():
|
|
bd = objects.BuildData(
|
|
node_name=node.name,
|
|
task_id=self.task.task_id,
|
|
collected_date=r.updated,
|
|
generator="{}:{}".format(stage, r.name),
|
|
data_format='text/plain',
|
|
data_element=r.get_decoded_data())
|
|
self.state_manager.post_build_data(bd)
|
|
log_href = "%s/tasks/%s/builddata" % (get_internal_api_href("v1.0"),
|
|
str(self.task.task_id))
|
|
self.task.result.add_link('detail_logs', log_href)
|
|
self.task.save()
|
|
|
|
|
|
class ValidateNodeServices(BaseMaasAction):
|
|
"""Action to validate MaaS is available and ready for use."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
|
|
try:
|
|
if self.maas_client.test_connectivity():
|
|
self.logger.info("Able to connect to MaaS.")
|
|
self.task.add_status_msg(
|
|
msg='Able to connect to MaaS.',
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
if self.maas_client.test_authentication():
|
|
self.logger.info("Able to authenticate with MaaS API.")
|
|
self.task.add_status_msg(
|
|
msg='Able to authenticate with MaaS API.',
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
|
|
boot_res = maas_boot_res.BootResources(self.maas_client)
|
|
boot_res.refresh()
|
|
|
|
if boot_res.is_importing():
|
|
self.logger.info(
|
|
"MaaS still importing boot resources.")
|
|
self.task.add_status_msg(
|
|
msg='MaaS still importing boot resources.',
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.failure()
|
|
else:
|
|
if boot_res.len() > 0:
|
|
self.logger.info("MaaS has synced boot resources.")
|
|
self.task.add_status_msg(
|
|
msg='MaaS has synced boot resources.',
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
else:
|
|
self.logger.info("MaaS has no boot resources.")
|
|
self.task.add_status_msg(
|
|
msg='MaaS has no boot resources.',
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.failure()
|
|
|
|
rack_ctlrs = maas_rack.RackControllers(self.maas_client)
|
|
rack_ctlrs.refresh()
|
|
|
|
if rack_ctlrs.len() == 0:
|
|
self.logger.info(
|
|
"No rack controllers registered in MaaS")
|
|
self.task.add_status_msg(
|
|
msg='No rack controllers registered in MaaS',
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.failure()
|
|
else:
|
|
healthy_rackd = []
|
|
for r in rack_ctlrs:
|
|
if r.is_healthy():
|
|
healthy_rackd.append(r.hostname)
|
|
else:
|
|
msg = "Rack controller %s not healthy." % r.hostname
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=r.hostname,
|
|
ctx_type='rack_ctlr')
|
|
if not healthy_rackd:
|
|
msg = "No healthy rack controllers found."
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx='maas',
|
|
ctx_type='cluster')
|
|
self.task.failure()
|
|
|
|
except errors.TransientDriverError as ex:
|
|
self.task.add_status_msg(
|
|
msg=str(ex), error=True, ctx='NA', ctx_type='NA', retry=True)
|
|
self.task.failure()
|
|
except errors.PersistentDriverError as ex:
|
|
self.task.add_status_msg(
|
|
msg=str(ex), error=True, ctx='NA', ctx_type='NA')
|
|
self.task.failure()
|
|
except Exception as ex:
|
|
self.task.add_status_msg(
|
|
msg=str(ex), error=True, ctx='NA', ctx_type='NA')
|
|
self.task.failure()
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class CreateStorageTemplate(BaseMaasAction):
|
|
"""Action for any site wide storage activity, not implemented."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class CreateBootMedia(BaseMaasAction):
|
|
"""Action to import image resources."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class PrepareHardwareConfig(BaseMaasAction):
|
|
"""Action to prepare commissioning configuration."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class InterrogateNode(BaseMaasAction):
|
|
"""Action to query MaaS for build-time information about the node."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class DestroyNode(BaseMaasAction):
|
|
"""Action to remove node from MaaS in preparation for redeploy."""
|
|
|
|
# define the list of node statuses, from which maas server allows releasing a node
|
|
|
|
# A machine can be released from following states, based on MaaS API reference.
|
|
# The disk of the released machine is erased, and the machine will end up in
|
|
# "Ready" state in MaaS after release.
|
|
actionable_node_statuses = (
|
|
"Allocated",
|
|
"Deployed",
|
|
"Deploying",
|
|
"Failed deployment",
|
|
"Releasing failed",
|
|
"Failed disk erasing",
|
|
)
|
|
|
|
def start(self):
|
|
"""
|
|
Destroy Node erases the storage, releases the BM node in MaaS, and
|
|
finally deletes the BM node as a resource from the MaaS database.
|
|
After successful completion of this action, the destroyed nodes are removed
|
|
from MaaS list of resources and will be Unkown to MaaS. These nodes have
|
|
to go through the enlistment process and be detected by MaaS as new nodes.
|
|
Destroy Node can be performed from any BM node state.
|
|
|
|
:return: None
|
|
"""
|
|
try:
|
|
machine_list = maas_machine.Machines(self.maas_client)
|
|
machine_list.refresh()
|
|
except Exception as ex:
|
|
self.logger.warning("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg='Error accessing MaaS Machines API: {}'.format(str(ex)),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
for n in nodes:
|
|
try:
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
|
|
if machine is None:
|
|
msg = "Could not locate machine for node {}".format(n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
continue
|
|
elif type(machine) == maas_rack.RackController:
|
|
msg = "Cannot delete rack controller {}.".format(n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
# First release the node and erase its disks, if MaaS API allows
|
|
if machine.status_name in self.actionable_node_statuses:
|
|
msg = "Releasing node {}, and erasing storage.".format(
|
|
n.name)
|
|
self.logger.info(msg)
|
|
|
|
try:
|
|
machine.release(erase_disk=True, quick_erase=True)
|
|
except errors.DriverError:
|
|
msg = "Error Releasing node {}, skipping".format(
|
|
n.name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
# node release with erase disk will take sometime monitor it
|
|
attempts = 0
|
|
max_attempts = (
|
|
config.config_mgr.conf.timeouts.destroy_node *
|
|
60) // config.config_mgr.conf.maasdriver.poll_interval
|
|
|
|
while (attempts < max_attempts and
|
|
(not machine.status_name.startswith('Ready')
|
|
and not machine.status_name.startswith('Failed'))):
|
|
attempts = attempts + 1
|
|
time.sleep(
|
|
config.config_mgr.conf.maasdriver.poll_interval)
|
|
try:
|
|
machine.refresh()
|
|
self.logger.debug(
|
|
"Polling node {} status attempt {:d} of {:d}: {}"
|
|
.format(n.name, attempts, max_attempts,
|
|
machine.status_name))
|
|
except Exception:
|
|
self.logger.warning(
|
|
"Error updating node {} status during release node, will re-attempt."
|
|
.format(n.name))
|
|
if machine.status_name.startswith('Ready'):
|
|
msg = "Node {} released and disk erased.".format(
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
else:
|
|
msg = "Node {} release timed out".format(n.name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
else:
|
|
# Node is in a state that cannot be released from MaaS API.
|
|
# Reset the storage instead
|
|
msg = "Destroy node {} in status: {}, resetting storage.".format(
|
|
n.name, machine.status_name)
|
|
self.logger.info(msg)
|
|
machine.reset_storage_config()
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
|
|
# for both cases above delete the node to force re-commissioning
|
|
# But, before deleting the node reset it power type in maas if
|
|
# the node power type should be virsh.
|
|
try:
|
|
if n.oob_type == 'libvirt':
|
|
self.logger.info(
|
|
'Resetting MaaS virsh power parameters for node {}.'
|
|
.format(n.name))
|
|
# setting power type attibutes to empty string
|
|
# will remove them from maas BMC table
|
|
machine.reset_power_parameters()
|
|
except AttributeError as attr_er:
|
|
pass
|
|
|
|
machine.delete()
|
|
msg = "Deleted Node: {} in status: {}.".format(
|
|
n.name, machine.status_name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
|
|
except errors.DriverError as dex:
|
|
msg = "Driver error, while destroying node {}, skipping".format(
|
|
n.name)
|
|
self.logger.warning(msg, exc_info=dex)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class CreateNetworkTemplate(BaseMaasAction):
|
|
"""Action for configuring the site wide network topology in MaaS."""
|
|
|
|
def start(self):
|
|
# TODO(sh8121att) Break this down into more functions
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
if not site_design.network_links:
|
|
msg = ("Site design has no network links, no work to do.")
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
self.task.success()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
# Try to true up MaaS definitions of fabrics/vlans/subnets
|
|
# with the networks defined in Drydock
|
|
design_networks = list()
|
|
design_links = site_design.network_links or []
|
|
|
|
fabrics = maas_fabric.Fabrics(self.maas_client)
|
|
fabrics.refresh()
|
|
|
|
subnets = maas_subnet.Subnets(self.maas_client)
|
|
subnets.refresh()
|
|
|
|
domains = maas_domain.Domains(self.maas_client)
|
|
domains.refresh()
|
|
|
|
for l in design_links:
|
|
if l.metalabels is not None:
|
|
# TODO(sh8121att): move metalabels into config
|
|
if 'noconfig' in l.metalabels:
|
|
self.logger.info(
|
|
"NetworkLink %s marked 'noconfig', skipping configuration including allowed networks."
|
|
% (l.name))
|
|
continue
|
|
|
|
fabrics_found = set()
|
|
|
|
# First loop through the possible Networks on this NetworkLink
|
|
# and validate that MaaS's self-discovered networking matches
|
|
# our design. This means all self-discovered networks that are matched
|
|
# to a link need to all be part of the same fabric. Otherwise there is no
|
|
# way to reconcile the discovered topology with the designed topology
|
|
for net_name in l.allowed_networks:
|
|
n = site_design.get_network(net_name)
|
|
|
|
if n is None:
|
|
msg = "Network %s allowed on link %s, but not defined." % (
|
|
net_name, l.name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=l.name,
|
|
ctx_type='network_link')
|
|
continue
|
|
|
|
maas_net = subnets.singleton({'cidr': n.cidr})
|
|
|
|
if maas_net is not None:
|
|
fabrics_found.add(maas_net.fabric)
|
|
|
|
if len(fabrics_found) > 1:
|
|
msg = "MaaS self-discovered network incompatible with NetworkLink %s" % l.name
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=l.name, ctx_type='network_link')
|
|
continue
|
|
elif len(fabrics_found) == 1:
|
|
link_fabric_id = fabrics_found.pop()
|
|
link_fabric = fabrics.select(link_fabric_id)
|
|
link_fabric.name = l.name
|
|
link_fabric.update()
|
|
else:
|
|
link_fabric = fabrics.singleton({'name': l.name})
|
|
|
|
if link_fabric is None:
|
|
link_fabric = maas_fabric.Fabric(
|
|
self.maas_client, name=l.name)
|
|
link_fabric = fabrics.add(link_fabric)
|
|
|
|
# Ensure that the MTU of the untagged VLAN on the fabric
|
|
# matches that on the NetworkLink config
|
|
|
|
vlan_list = maas_vlan.Vlans(
|
|
self.maas_client, fabric_id=link_fabric.resource_id)
|
|
vlan_list.refresh()
|
|
msg = "Updating native VLAN MTU = %d on network link %s" % (l.mtu,
|
|
l.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=l.name, ctx_type='network_link')
|
|
vlan = vlan_list.singleton({'vid': 0})
|
|
if vlan:
|
|
vlan.mtu = l.mtu
|
|
vlan.update()
|
|
else:
|
|
self.logger.warning("Unable to find native VLAN on fabric %s."
|
|
% link_fabric.resource_id)
|
|
|
|
# Now that we have the fabrics sorted out, check
|
|
# that VLAN tags and subnet attributes are correct
|
|
for net_name in l.allowed_networks:
|
|
n = site_design.get_network(net_name)
|
|
design_networks.append(n)
|
|
|
|
if n is None:
|
|
continue
|
|
|
|
try:
|
|
domain = domains.singleton({'name': n.dns_domain})
|
|
|
|
if not domain:
|
|
self.logger.info(
|
|
'Network domain not found, adding: %s',
|
|
n.dns_domain)
|
|
domain = maas_domain.Domain(
|
|
self.maas_client,
|
|
name=n.dns_domain,
|
|
authoritative=False)
|
|
domain = domains.add(domain)
|
|
domains.refresh()
|
|
|
|
subnet = subnets.singleton({'cidr': n.cidr})
|
|
|
|
if subnet is None:
|
|
msg = "Subnet for network %s not found, creating..." % (
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
|
|
fabric_list = maas_fabric.Fabrics(self.maas_client)
|
|
fabric_list.refresh()
|
|
fabric = fabric_list.singleton({'name': l.name})
|
|
|
|
if fabric is not None:
|
|
vlan_list = maas_vlan.Vlans(
|
|
self.maas_client, fabric_id=fabric.resource_id)
|
|
vlan_list.refresh()
|
|
|
|
vlan = vlan_list.singleton({
|
|
'vid':
|
|
n.vlan_id if n.vlan_id is not None else 0
|
|
})
|
|
|
|
if vlan is not None:
|
|
vlan.name = n.name
|
|
|
|
if getattr(n, 'mtu', None) is not None:
|
|
vlan.mtu = n.mtu
|
|
|
|
vlan.update()
|
|
msg = "VLAN %s found for network %s, updated attributes" % (
|
|
vlan.resource_id, n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
else:
|
|
# Create a new VLAN in this fabric and assign subnet to it
|
|
vlan = maas_vlan.Vlan(
|
|
self.maas_client,
|
|
name=n.name,
|
|
vid=n.vlan_id,
|
|
mtu=getattr(n, 'mtu', None),
|
|
fabric_id=fabric.resource_id)
|
|
vlan = vlan_list.add(vlan)
|
|
|
|
msg = "VLAN %s created for network %s" % (
|
|
vlan.resource_id, n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
|
|
# If subnet did not exist, create it here and attach it to the fabric/VLAN
|
|
subnet = maas_subnet.Subnet(
|
|
self.maas_client,
|
|
name=n.name,
|
|
cidr=n.cidr,
|
|
dns_servers=n.dns_servers,
|
|
fabric=fabric.resource_id,
|
|
vlan=vlan.resource_id,
|
|
gateway_ip=n.get_default_gateway())
|
|
|
|
subnet_list = maas_subnet.Subnets(self.maas_client)
|
|
subnet = subnet_list.add(subnet)
|
|
|
|
msg = "Created subnet %s for CIDR %s on VLAN %s" % (
|
|
subnet.resource_id, subnet.cidr, subnet.vlan)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
else:
|
|
msg = "Fabric %s should be created, but cannot locate it." % (
|
|
l.name)
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='network_link')
|
|
else:
|
|
subnet.name = n.name
|
|
subnet.dns_servers = n.dns_servers
|
|
|
|
msg = "Subnet %s found for network %s, updated attributes" % (
|
|
subnet.resource_id, n.name)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
self.logger.info(msg)
|
|
|
|
vlan_list = maas_vlan.Vlans(
|
|
self.maas_client, fabric_id=subnet.fabric)
|
|
vlan_list.refresh()
|
|
|
|
vlan = vlan_list.select(subnet.vlan)
|
|
|
|
if vlan is not None:
|
|
vlan.name = n.name
|
|
vlan.set_vid(n.vlan_id)
|
|
|
|
if getattr(n, 'mtu', None) is not None:
|
|
vlan.mtu = n.mtu
|
|
|
|
vlan.update()
|
|
msg = "VLAN %s found for network %s, updated attributes" % (
|
|
vlan.resource_id, n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
else:
|
|
msg = "MaaS subnet %s does not have a matching VLAN" % (
|
|
subnet.resource_id)
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
self.task.failure(focus=n.name)
|
|
|
|
# Check if the routes have a default route
|
|
subnet.gateway_ip = n.get_default_gateway()
|
|
subnet.update()
|
|
|
|
dhcp_on = False
|
|
|
|
for r in n.ranges:
|
|
try:
|
|
subnet.add_address_range(r)
|
|
if r.get('type', None) == 'dhcp':
|
|
dhcp_on = True
|
|
except Exception as e:
|
|
msg = "Error adding range to network %s: %s" % (
|
|
n.name, str(r))
|
|
self.logger.error(msg, exc_info=e)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
|
|
vlan_list = maas_vlan.Vlans(
|
|
self.maas_client, fabric_id=subnet.fabric)
|
|
vlan_list.refresh()
|
|
vlan = vlan_list.select(subnet.vlan)
|
|
|
|
if dhcp_on:
|
|
# check if design requires a dhcp relay and if the MaaS vlan already uses a dhcp_relay
|
|
msg = "DHCP enabled for subnet %s, activating in MaaS" % (
|
|
subnet.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
|
|
rack_ctlrs = maas_rack.RackControllers(
|
|
self.maas_client)
|
|
rack_ctlrs.refresh()
|
|
|
|
# Reset DHCP stuff to avoid offline rack controllers
|
|
|
|
vlan.reset_dhcp_mgmt()
|
|
dhcp_config_set = False
|
|
|
|
for r in rack_ctlrs:
|
|
if n.dhcp_relay_upstream_target is not None:
|
|
if r.interface_for_ip(
|
|
n.dhcp_relay_upstream_target):
|
|
if not r.is_healthy():
|
|
msg = ("Rack controller %s with DHCP relay is not healthy." %
|
|
r.hostname)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
break
|
|
iface = r.interface_for_ip(
|
|
n.dhcp_relay_upstream_target)
|
|
vlan.relay_vlan = iface.vlan
|
|
msg = "Relaying DHCP on vlan %s to vlan %s" % (
|
|
vlan.resource_id, vlan.relay_vlan)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
vlan.update()
|
|
dhcp_config_set = True
|
|
break
|
|
else:
|
|
for i in r.interfaces:
|
|
if i.vlan == vlan.resource_id:
|
|
msg = "Rack controller %s has interface on vlan %s" % (
|
|
r.resource_id, vlan.resource_id)
|
|
self.logger.debug(msg)
|
|
rackctl_id = r.resource_id
|
|
|
|
if not r.is_healthy():
|
|
msg = ("Rack controller %s not healthy, skipping DHCP config." %
|
|
r.resource_id)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
break
|
|
try:
|
|
vlan.dhcp_on = True
|
|
vlan.add_rack_controller(
|
|
rackctl_id)
|
|
msg = "Enabling DHCP on VLAN %s managed by rack ctlr %s" % (
|
|
vlan.resource_id, rackctl_id)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
vlan.update()
|
|
dhcp_config_set = True
|
|
except RackControllerConflict as rack_ex:
|
|
msg = (
|
|
"More than two rack controllers on vlan %s, "
|
|
"skipping enabling %s." %
|
|
(vlan.resource_id, rackctl_id))
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
break
|
|
|
|
if not dhcp_config_set:
|
|
msg = "Network %s requires DHCP, but could not locate a rack controller to serve it." % (
|
|
n.name)
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='network')
|
|
self.task.failure(focus=n.name)
|
|
|
|
except ValueError:
|
|
raise errors.DriverError("Inconsistent data from MaaS")
|
|
|
|
# Now that all networks should be created, we can setup routes between them
|
|
subnet_list = maas_subnet.Subnets(self.maas_client)
|
|
subnet_list.refresh()
|
|
|
|
for n in design_networks:
|
|
src_subnet = subnet_list.singleton({'cidr': n.cidr})
|
|
for r in n.routes:
|
|
# care for case of routedomain routes that are logical placeholders
|
|
if not r.get('subnet', None):
|
|
continue
|
|
route_net = r.get('subnet')
|
|
# Skip the default route case
|
|
if route_net == '0.0.0.0/0':
|
|
continue
|
|
dest_subnet = subnet_list.singleton({'cidr': route_net})
|
|
if dest_subnet is not None:
|
|
src_subnet.add_static_route(
|
|
dest_subnet.resource_id,
|
|
r.get('gateway'),
|
|
metric=r.get('metric', 100))
|
|
else:
|
|
msg = "Could not locate destination network for static route to %s." % route_net
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='network')
|
|
self.task.failure(focus=n.name)
|
|
self.logger.info(msg)
|
|
continue
|
|
|
|
# now validate that all subnets allowed on a link were created
|
|
|
|
for n in design_networks:
|
|
if n.metalabels is not None:
|
|
# TODO(sh8121att): move metalabels into config
|
|
if 'noconfig' in n.metalabels:
|
|
self.logger.info(
|
|
"Network %s marked 'noconfig', skipping validation." %
|
|
(n.name))
|
|
continue
|
|
|
|
exists = subnet_list.singleton({'cidr': n.cidr})
|
|
if exists is not None:
|
|
self.task.success(focus=n.name)
|
|
else:
|
|
msg = "Network %s defined, but not found in MaaS after network config task." % n.name
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='network')
|
|
self.task.failure(focus=n.name)
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class ConfigureNodeProvisioner(BaseMaasAction):
|
|
"""Action for configuring site-wide node provisioner options."""
|
|
# These repo names cannot be deleted out of MAAS
|
|
# and maintain functionality
|
|
DEFAULT_REPOS = ['main_archive', 'ports_archive']
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
try:
|
|
current_repos = maas_repo.Repositories(self.maas_client)
|
|
current_repos.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg='Error accessing MaaS SshKeys API',
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
site_model = site_design.get_site()
|
|
|
|
repo_list = getattr(site_model, 'repositories', None) or []
|
|
|
|
if repo_list:
|
|
for r in repo_list:
|
|
try:
|
|
existing_repo = current_repos.singleton({
|
|
'name': r.get_id()
|
|
})
|
|
new_repo = self.create_maas_repo(self.maas_client, r)
|
|
if existing_repo:
|
|
new_repo.resource_id = existing_repo.resource_id
|
|
new_repo.update()
|
|
msg = "Updating repository definition for %s." % (
|
|
r.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
self.task.success()
|
|
else:
|
|
new_repo = current_repos.add(new_repo)
|
|
msg = "Adding repository definition for %s." % (r.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
self.task.success()
|
|
except Exception as ex:
|
|
msg = "Error adding repository to MaaS configuration: %s" % str(
|
|
ex)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx='NA', ctx_type='NA')
|
|
self.task.failure()
|
|
if repo_list.remove_unlisted:
|
|
defined_repos = [x.get_id() for x in repo_list]
|
|
to_delete = [
|
|
r for r in current_repos if r.name not in defined_repos
|
|
]
|
|
for r in to_delete:
|
|
if r.name not in self.DEFAULT_REPOS:
|
|
r.delete()
|
|
current_repos.refresh()
|
|
else:
|
|
msg = ("No repositories to add, no work to do.")
|
|
self.logger.debug(msg)
|
|
self.task.success()
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
@staticmethod
|
|
def create_maas_repo(api_client, repo_obj):
|
|
"""Create a MAAS model of a repo based of a Drydock model.
|
|
|
|
If resource_id is specified, assign it the resource_id so the new instance
|
|
can be used to update an existing repo.
|
|
|
|
:param api_client: A MAAS API client configured to connect to MAAS
|
|
:param repo_obj: Instance of objects.Repository
|
|
"""
|
|
model_fields = dict()
|
|
if repo_obj.distributions:
|
|
model_fields['distributions'] = ','.join(repo_obj.distributions)
|
|
if repo_obj.components:
|
|
if repo_obj.get_id() in ConfigureNodeProvisioner.DEFAULT_REPOS:
|
|
model_fields['disabled_components'] = ','.join(
|
|
repo_obj.get_disabled_components())
|
|
else:
|
|
model_fields['components'] = ','.join(repo_obj.components)
|
|
if repo_obj.get_disabled_subrepos():
|
|
model_fields['disabled_pockets'] = ','.join(
|
|
repo_obj.get_disabled_subrepos())
|
|
if repo_obj.arches:
|
|
model_fields['arches'] = ','.join(repo_obj.arches)
|
|
|
|
model_fields['key'] = repo_obj.gpgkey
|
|
|
|
for k in ['name', 'url']:
|
|
model_fields[k] = getattr(repo_obj, k)
|
|
|
|
repo_model = maas_repo.Repository(api_client, **model_fields)
|
|
return repo_model
|
|
|
|
|
|
class ConfigureUserCredentials(BaseMaasAction):
|
|
"""Action for configuring user public keys."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
try:
|
|
current_keys = maas_keys.SshKeys(self.maas_client)
|
|
current_keys.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg='Error accessing MaaS SshKeys API',
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
site_model = site_design.get_site()
|
|
|
|
key_list = getattr(site_model, 'authorized_keys', None) or []
|
|
|
|
if key_list:
|
|
for k in key_list:
|
|
try:
|
|
if len(current_keys.query({
|
|
'key': k.replace("\n", "")
|
|
})) == 0:
|
|
new_key = maas_keys.SshKey(self.maas_client, key=k)
|
|
new_key = current_keys.add(new_key)
|
|
msg = "Added SSH key %s to MaaS user profile. Will be installed on all deployed nodes." % (
|
|
k[:16])
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
self.task.success()
|
|
else:
|
|
msg = "SSH key %s already exists in MaaS user profile." % k[:
|
|
16]
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
self.task.success()
|
|
except Exception as ex:
|
|
msg = "Error adding SSH key to MaaS user profile: %s" % str(
|
|
ex)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx='NA', ctx_type='NA')
|
|
self.task.failure()
|
|
else:
|
|
msg = ("No keys to add, no work to do.")
|
|
self.logger.debug(msg)
|
|
self.task.success()
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx='NA', ctx_type='NA')
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class IdentifyNode(BaseMaasAction):
|
|
"""Action to identify a node resource in MaaS matching a node design."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
|
|
for n in nodes:
|
|
try:
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
if machine is None:
|
|
self.task.failure(focus=n.get_id())
|
|
self.task.add_status_msg(
|
|
msg="Node %s not found in MaaS" % n.name,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
elif type(machine) == maas_machine.Machine:
|
|
machine.update_identity(n, domain=n.get_domain(site_design))
|
|
msg = "Node %s identified in MaaS" % n.name
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
elif type(machine) == maas_rack.RackController:
|
|
msg = "Rack controller %s identified in MaaS" % n.name
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
except ApiNotAvailable as api_ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=api_ex)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg='Error accessing MaaS API: %s' % str(api_ex),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
except Exception as ex:
|
|
self.logger.debug(
|
|
"Exception caught in identify node.", exc_info=ex)
|
|
self.task.failure(focus=n.get_id())
|
|
self.task.add_status_msg(
|
|
msg="Error trying to location %s in MAAS" % n.name,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
class ConfigureHardware(BaseMaasAction):
|
|
"""Action to start commissioning a server."""
|
|
|
|
def start(self):
|
|
try:
|
|
machine_list = maas_machine.Machines(self.maas_client)
|
|
machine_list.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg="Error accessing MaaS Machines API: %s" % str(ex),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
|
|
# TODO(sh8121att): Better way of representing the node statuses than static strings
|
|
for n in nodes:
|
|
try:
|
|
self.logger.debug(
|
|
"Locating node %s for commissioning" % (n.name))
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
if type(machine) == maas_rack.RackController:
|
|
msg = "Located node %s in MaaS as rack controller. Skipping." % (
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
elif machine is not None:
|
|
if machine.status_name in [
|
|
'New', 'Broken', 'Failed commissioning',
|
|
'Failed testing'
|
|
]:
|
|
self.logger.debug(
|
|
"Located node %s in MaaS, starting commissioning" %
|
|
(n.name))
|
|
machine.commission()
|
|
|
|
# Poll machine status
|
|
attempts = 0
|
|
max_attempts = (
|
|
config.config_mgr.conf.timeouts.configure_hardware
|
|
* 60
|
|
) // config.config_mgr.conf.maasdriver.poll_interval
|
|
|
|
while (attempts < max_attempts and
|
|
(machine.status_name != 'Ready' and
|
|
not machine.status_name.startswith('Failed'))):
|
|
attempts = attempts + 1
|
|
time.sleep(config.config_mgr.conf.maasdriver.
|
|
poll_interval)
|
|
try:
|
|
machine.refresh()
|
|
self.logger.debug(
|
|
"Polling node %s status attempt %d of %d: %s"
|
|
% (n.name, attempts, max_attempts,
|
|
machine.status_name))
|
|
except Exception:
|
|
self.logger.warning(
|
|
"Error updating node %s status during commissioning, will re-attempt."
|
|
% (n.name),
|
|
exc_info=True)
|
|
if machine.status_name == 'Ready':
|
|
msg = "Node %s commissioned." % (n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
self.collect_build_data(machine)
|
|
else:
|
|
msg = "Node %s failed commissioning." % (n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
self._add_detail_logs(
|
|
n,
|
|
machine,
|
|
'commission',
|
|
result_type='commissioning')
|
|
self._add_detail_logs(
|
|
n, machine, 'testing', result_type='testing')
|
|
elif machine.status_name in ['Commissioning', 'Testing']:
|
|
msg = "Located node %s in MaaS, node already being commissioned. Skipping..." % (
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
elif machine.status_name in [
|
|
'Ready', 'Deploying', 'Allocated', 'Deployed'
|
|
]:
|
|
msg = "Located node %s in MaaS, node commissioned. Skipping..." % (
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
else:
|
|
msg = "Located node %s in MaaS, unknown status %s. Skipping." % (
|
|
n, machine.status_name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
else:
|
|
msg = "Node %s not found in MaaS" % n.name
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
except Exception as ex:
|
|
msg = "Error commissioning node %s: %s" % (n.name, str(ex))
|
|
self.logger.warning(msg)
|
|
self.logger.debug(
|
|
"Unhandled exception attempting to commission node.",
|
|
exc_info=ex)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
def collect_build_data(self, machine):
|
|
"""Collect MaaS build data after commissioning."""
|
|
self.logger.debug("Collecting build data for %s" % machine.hostname)
|
|
try:
|
|
data = machine.get_details()
|
|
if data:
|
|
for t, d in data.items():
|
|
if t in ('lshw', 'lldp'):
|
|
df = 'text/xml'
|
|
else:
|
|
df = 'text/plain'
|
|
bd = objects.BuildData(
|
|
node_name=machine.hostname,
|
|
task_id=self.task.get_id(),
|
|
generator=t,
|
|
collected_date=datetime.utcnow(),
|
|
data_format=df,
|
|
data_element=d.decode())
|
|
self.logger.debug(
|
|
"Saving build data from generator %s" % t)
|
|
self.state_manager.post_build_data(bd)
|
|
self.task.add_status_msg(
|
|
msg="Saving build data element.",
|
|
error=False,
|
|
ctx=machine.hostname,
|
|
ctx_type='node')
|
|
except Exception as ex:
|
|
self.logger.error(
|
|
"Error collecting node build data for %s" % machine.hostname,
|
|
exc_info=ex)
|
|
|
|
|
|
class ApplyNodeNetworking(BaseMaasAction):
|
|
"""Action to configure networking on a node."""
|
|
|
|
def start(self):
|
|
try:
|
|
machine_list = maas_machine.Machines(self.maas_client)
|
|
machine_list.refresh()
|
|
|
|
fabrics = maas_fabric.Fabrics(self.maas_client)
|
|
fabrics.refresh()
|
|
|
|
subnets = maas_subnet.Subnets(self.maas_client)
|
|
subnets.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg="Error accessing MaaS API: %s" % str(ex),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design(resolve_aliases=True)
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
|
|
# TODO(sh8121att): Better way of representing the node statuses than static strings
|
|
for n in nodes:
|
|
try:
|
|
self.logger.debug(
|
|
"Locating node %s for network configuration" % (n.name))
|
|
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
|
|
if type(machine) is maas_rack.RackController:
|
|
msg = ("Node %s is a rack controller, skipping deploy action." %
|
|
n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
elif machine is not None:
|
|
if machine.status_name.startswith('Failed Dep'):
|
|
msg = (
|
|
"Node %s has failed deployment, releasing to try again."
|
|
% n.name)
|
|
self.logger.debug(msg)
|
|
try:
|
|
machine.release()
|
|
machine.refresh()
|
|
except errors.DriverError as ex:
|
|
msg = (
|
|
"Node %s could not be released, skipping deployment."
|
|
% n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
continue
|
|
msg = ("Released failed node %s to retry deployment." %
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
|
|
if machine.status_name == 'Ready':
|
|
msg = "Located node %s in MaaS, starting interface configuration" % (
|
|
n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
|
|
machine.reset_network_config()
|
|
machine.refresh()
|
|
|
|
for i in n.interfaces:
|
|
if not i.network_link:
|
|
self.logger.debug(
|
|
"Interface %s has no network link, skipping configuration."
|
|
% (i.device_name))
|
|
continue
|
|
|
|
nl = site_design.get_network_link(i.network_link)
|
|
|
|
if nl.metalabels is not None:
|
|
if 'noconfig' in nl.metalabels:
|
|
self.logger.info(
|
|
"Interface %s connected to NetworkLink %s marked 'noconfig', skipping."
|
|
% (i.device_name, nl.name))
|
|
continue
|
|
|
|
fabric = fabrics.singleton({'name': nl.name})
|
|
|
|
if fabric is None:
|
|
msg = "No fabric found for NetworkLink %s" % (
|
|
nl.name)
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
continue
|
|
|
|
if nl.bonding_mode != hd_fields.NetworkLinkBondingMode.Disabled:
|
|
if len(i.get_hw_slaves()) > 1:
|
|
msg = "Building node %s interface %s as a bond." % (
|
|
n.name, i.device_name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
hw_iface_list = i.get_hw_slaves()
|
|
hw_iface_logicalname_list = []
|
|
for hw_iface in hw_iface_list:
|
|
hw_iface_logicalname_list.append(
|
|
n.get_logicalname(hw_iface))
|
|
iface = machine.interfaces.create_bond(
|
|
device_name=i.device_name,
|
|
parent_names=hw_iface_logicalname_list,
|
|
mtu=nl.mtu,
|
|
fabric=fabric.resource_id,
|
|
mode=nl.bonding_mode,
|
|
monitor_interval=nl.bonding_mon_rate,
|
|
downdelay=nl.bonding_down_delay,
|
|
updelay=nl.bonding_up_delay,
|
|
lacp_rate=nl.bonding_peer_rate,
|
|
hash_policy=nl.bonding_xmit_hash)
|
|
else:
|
|
msg = "Network link %s indicates bonding, " \
|
|
"interface %s has less than 2 slaves." % \
|
|
(nl.name, i.device_name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
continue
|
|
else:
|
|
if len(i.get_hw_slaves()) > 1:
|
|
msg = "Network link %s disables bonding, interface %s has multiple slaves." % \
|
|
(nl.name, i.device_name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(n.name)
|
|
continue
|
|
elif len(i.get_hw_slaves()) == 0:
|
|
msg = "Interface %s has 0 slaves." % (
|
|
i.device_name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
else:
|
|
msg = "Configuring interface %s on node %s" % (
|
|
i.device_name, n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
hw_iface = i.get_hw_slaves()[0]
|
|
# TODO(sh8121att): HardwareProfile device alias integration
|
|
iface = machine.get_network_interface(
|
|
n.get_logicalname(hw_iface))
|
|
|
|
if iface is None:
|
|
msg = "Interface %s not found on node %s, skipping configuration" % (
|
|
i.device_name, machine.resource_id)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
continue
|
|
|
|
if iface.fabric_id == fabric.resource_id:
|
|
self.logger.debug(
|
|
"Interface %s already attached to fabric_id %s"
|
|
% (i.device_name, fabric.resource_id))
|
|
else:
|
|
self.logger.debug(
|
|
"Attaching node %s interface %s to fabric_id %s"
|
|
% (n.name, i.device_name,
|
|
fabric.resource_id))
|
|
iface.attach_fabric(
|
|
fabric_id=fabric.resource_id)
|
|
|
|
if iface.effective_mtu != nl.mtu:
|
|
self.logger.debug(
|
|
"Updating interface %s MTU to %s" %
|
|
(i.device_name, nl.mtu))
|
|
iface.set_mtu(nl.mtu)
|
|
|
|
for iface_net in getattr(i, 'networks', []):
|
|
dd_net = site_design.get_network(iface_net)
|
|
|
|
if dd_net is not None:
|
|
link_iface = None
|
|
if iface_net == getattr(
|
|
nl, 'native_network', None):
|
|
# If a node interface is attached to the native network for a link
|
|
# then the interface itself should be linked to network, not a VLAN
|
|
# tagged interface
|
|
self.logger.debug(
|
|
"Attaching node %s interface %s to untagged VLAN on fabric %s"
|
|
% (n.name, i.device_name,
|
|
fabric.resource_id))
|
|
link_iface = iface
|
|
else:
|
|
# For non-native networks, we create VLAN tagged interfaces as children
|
|
# of this interface
|
|
vlan_options = {
|
|
'vlan_tag': dd_net.vlan_id,
|
|
'parent_name': iface.name,
|
|
}
|
|
|
|
if dd_net.mtu is not None:
|
|
vlan_options['mtu'] = dd_net.mtu
|
|
|
|
self.logger.debug(
|
|
"Creating tagged interface for VLAN %s on system %s interface %s"
|
|
% (dd_net.vlan_id, n.name,
|
|
i.device_name))
|
|
|
|
try:
|
|
link_iface = machine.interfaces.create_vlan(
|
|
**vlan_options)
|
|
except errors.DriverError as ex:
|
|
msg = "Error creating interface: %s" % str(
|
|
ex)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
continue
|
|
|
|
link_options = {}
|
|
link_options[
|
|
'primary'] = True if iface_net == getattr(
|
|
n, 'primary_network',
|
|
None) else False
|
|
link_options['subnet_cidr'] = dd_net.cidr
|
|
|
|
found = False
|
|
for a in getattr(n, 'addressing', []):
|
|
if a.network == iface_net:
|
|
link_options[
|
|
'ip_address'] = 'dhcp' if a.type == 'dhcp' else a.address
|
|
found = True
|
|
|
|
if not found:
|
|
msg = "No addressed assigned to network %s for node %s, link is L2 only." % (
|
|
iface_net, n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
link_options['ip_address'] = None
|
|
|
|
msg = "Linking system %s interface %s to subnet %s" % (
|
|
n.name, i.device_name, dd_net.cidr)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
|
|
link_iface.link_subnet(**link_options)
|
|
self.task.success(focus=n.name)
|
|
else:
|
|
self.task.failure(focus=n.name)
|
|
msg = "Did not find a defined Network %s to attach to interface" % iface_net
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
elif machine.status_name == 'Broken':
|
|
msg = ("Located node %s in MaaS, status broken. Run "
|
|
"ConfigureHardware before configurating network"
|
|
% (n.name))
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
elif machine.status_name == 'Deployed':
|
|
msg = (
|
|
"Located node %s in MaaS, status deployed. Skipping "
|
|
"and considering success. Destroy node first if redeploy needed."
|
|
% (n.name))
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
|
|
else:
|
|
msg = "Located node %s in MaaS, unknown status %s. Skipping..." % (
|
|
n.name, machine.status_name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
else:
|
|
msg = "Node %s not found in MaaS" % n.name
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure()
|
|
except Exception as ex:
|
|
msg = "Error configuring network for node %s: %s" % (n.name,
|
|
str(ex))
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.name)
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
|
|
|
|
class ApplyNodePlatform(BaseMaasAction):
|
|
"""Action for configuring platform settings (OS/kernel) on a node."""
|
|
|
|
def start(self):
|
|
try:
|
|
machine_list = maas_machine.Machines(self.maas_client)
|
|
machine_list.refresh()
|
|
|
|
tag_list = maas_tag.Tags(self.maas_client)
|
|
tag_list.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg="Error accessing MaaS API: %s" % str(ex),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
|
|
for n in nodes:
|
|
try:
|
|
self.logger.debug(
|
|
"Locating node %s for platform configuration" % (n.name))
|
|
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
|
|
if machine is None:
|
|
msg = "Could not locate machine for node %s" % n.name
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
except Exception as ex1:
|
|
msg = "Error locating machine for node %s: %s" % (n, str(ex1))
|
|
self.task.failure(focus=n.get_id())
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
continue
|
|
|
|
if type(machine) is maas_rack.RackController:
|
|
msg = ("Skipping changes to rack controller %s." % n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
elif machine.status_name == 'Deployed':
|
|
msg = (
|
|
"Located node %s in MaaS, status deployed. Skipping "
|
|
"and considering success. Destroy node first if redeploy needed."
|
|
% (n.name))
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
|
|
try:
|
|
# Render the string of all kernel params for the node
|
|
kp_string = n.get_kernel_param_string()
|
|
|
|
if kp_string:
|
|
# Check if the node has an existing kernel params tag
|
|
node_kp_tag = tag_list.select("%s_kp" % (n.name))
|
|
self.logger.info(
|
|
"Configuring kernel parameters for node %s" % (n.name))
|
|
|
|
if node_kp_tag:
|
|
node_kp_tag.delete()
|
|
|
|
msg = "Creating kernel_params tag for node %s: %s" % (
|
|
n.name, kp_string)
|
|
self.logger.debug(msg)
|
|
node_kp_tag = maas_tag.Tag(
|
|
self.maas_client,
|
|
name="%s_kp" % (n.name),
|
|
kernel_opts=kp_string)
|
|
node_kp_tag = tag_list.add(node_kp_tag)
|
|
node_kp_tag.apply_to_node(machine.resource_id)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
|
|
msg = "Applied kernel parameters to node %s" % n.name
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
else:
|
|
msg = "No kernel parameters to apply for %s." % n.name
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
except Exception as ex2:
|
|
msg = "Error configuring kernel parameters for node %s" % (
|
|
n.name)
|
|
self.logger.error(msg + ": %s" % str(ex2))
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
try:
|
|
if n.tags is not None and len(n.tags) > 0:
|
|
self.logger.info(
|
|
"Configuring static tags for node %s" % (n.name))
|
|
|
|
for t in n.tags:
|
|
tag_list.refresh()
|
|
tag = tag_list.select(t)
|
|
|
|
if tag is None:
|
|
try:
|
|
self.logger.debug("Creating static tag %s" % t)
|
|
tag = maas_tag.Tag(self.maas_client, name=t)
|
|
tag = tag_list.add(tag)
|
|
except errors.DriverError:
|
|
tag_list.refresh()
|
|
tag = tag_list.select(t)
|
|
if tag is not None:
|
|
self.logger.debug(
|
|
"Tag %s arrived out of nowhere." % t)
|
|
else:
|
|
self.logger.error(
|
|
"Error creating tag %s." % t)
|
|
continue
|
|
|
|
msg = "Applying tag %s to node %s" % (
|
|
tag.resource_id, machine.resource_id)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
tag.apply_to_node(machine.resource_id)
|
|
self.logger.info(
|
|
"Applied static tags to node %s" % (n.name))
|
|
self.task.success(focus=n.get_id())
|
|
else:
|
|
msg = "No node tags to apply for %s." % n.name
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
except Exception as ex3:
|
|
msg = "Error configuring static tags for node %s" % (n.name)
|
|
self.logger.error(msg + ": " + str(ex3))
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class ApplyNodeStorage(BaseMaasAction):
|
|
"""Action configure node storage."""
|
|
|
|
# NOTE(sh8121att) Partition tables take size and it seems if the first partition
|
|
# on a block device attempts to use the full size of the device
|
|
# MAAS will fail that the device is not large enough
|
|
# It may be that the block device available_size does not include overhead
|
|
# for the partition table and once the table is written, there is not
|
|
# enough space for the 'full size' partition. So reserve the below
|
|
# when calculating 'rest of device' sizing w/ the '>' operator
|
|
#
|
|
# This size is based on documentation that for backwards compatability
|
|
# the first partition should start on LBA 63 and we'll assume 4096 byte
|
|
# blocks, thus 63 (add one for safety) x 4096 = 258048
|
|
PART_TABLE_RESERVATION = 258048
|
|
|
|
def start(self):
|
|
try:
|
|
machine_list = maas_machine.Machines(self.maas_client)
|
|
machine_list.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg="Error accessing MaaS API: %s" % str(ex),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design(resolve_aliases=True)
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
|
|
for n in nodes:
|
|
try:
|
|
self.logger.debug(
|
|
"Locating node %s for storage configuration" % (n.name))
|
|
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
|
|
if machine is None:
|
|
msg = "Could not locate machine for node %s" % n.name
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
except Exception as ex:
|
|
msg = "Error locating machine for node %s" % (n.name)
|
|
self.logger.error(msg + ": " + str(ex))
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
if type(machine) is maas_rack.RackController:
|
|
msg = ("Skipping configuration updates to rack controller %s." %
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
elif machine.status_name == 'Deployed':
|
|
msg = (
|
|
"Located node %s in MaaS, status deployed. Skipping "
|
|
"and considering success. Destroy node first if redeploy needed."
|
|
% (n.name))
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
|
|
try:
|
|
"""
|
|
1. Clear VGs
|
|
2. Clear partitions
|
|
3. Apply partitioning
|
|
4. Create VGs
|
|
5. Create logical volumes
|
|
"""
|
|
msg = "Clearing current storage layout on node %s." % n.name
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
machine.reset_storage_config()
|
|
|
|
(root_dev, root_block) = n.find_fs_block_device('/')
|
|
(boot_dev, boot_block) = n.find_fs_block_device('/boot')
|
|
|
|
storage_layout = dict()
|
|
if isinstance(root_block, hostprofile.HostPartition):
|
|
storage_layout['layout_type'] = 'flat'
|
|
storage_layout['root_device'] = n.get_logicalname(
|
|
root_dev.name)
|
|
storage_layout['root_size'] = root_block.size
|
|
elif isinstance(root_block, hostprofile.HostVolume):
|
|
storage_layout['layout_type'] = 'lvm'
|
|
if len(root_dev.physical_devices) != 1:
|
|
msg = "Root LV in VG with multiple physical devices on node %s" % (
|
|
n.name)
|
|
self.logger.error(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
storage_layout['root_device'] = n.get_logicalname(
|
|
root_dev.physical_devices[0])
|
|
storage_layout['root_lv_size'] = root_block.size
|
|
storage_layout['root_lv_name'] = root_block.name
|
|
storage_layout['root_vg_name'] = root_dev.name
|
|
|
|
if boot_block is not None:
|
|
storage_layout['boot_size'] = boot_block.size
|
|
|
|
msg = "Setting node %s root storage layout: %s" % (
|
|
n.name, str(storage_layout))
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
machine.set_storage_layout(**storage_layout)
|
|
vg_devs = {}
|
|
|
|
for d in n.storage_devices:
|
|
maas_dev = machine.block_devices.singleton({
|
|
'name':
|
|
n.get_logicalname(d.name)
|
|
})
|
|
if maas_dev is None:
|
|
self.logger.warning(
|
|
"Dev %s (%s) not found on node %s" %
|
|
(d.name, n.get_logicalname(d.name), n.name))
|
|
continue
|
|
|
|
if d.volume_group is not None:
|
|
self.logger.debug(
|
|
"Adding dev %s (%s) to volume group %s" %
|
|
(d.name, n.get_logicalname(d.name),
|
|
d.volume_group))
|
|
if d.volume_group not in vg_devs:
|
|
vg_devs[d.volume_group] = {'b': [], 'p': []}
|
|
vg_devs[d.volume_group]['b'].append(
|
|
maas_dev.resource_id)
|
|
continue
|
|
|
|
self.logger.debug(
|
|
"Partitioning dev %s (%s) on node %s" %
|
|
(d.name, n.get_logicalname(d.name), n.name))
|
|
for p in d.partitions:
|
|
if p.is_sys():
|
|
self.logger.debug(
|
|
"Skipping manually configuring a system partition."
|
|
)
|
|
continue
|
|
maas_dev.refresh()
|
|
size = ApplyNodeStorage.calculate_bytes(
|
|
size_str=p.size, context=maas_dev)
|
|
part = maas_partition.Partition(
|
|
self.maas_client, size=size, bootable=p.bootable)
|
|
if p.part_uuid is not None:
|
|
part.uuid = p.part_uuid
|
|
msg = "Creating partition %s sized %d bytes on dev %s (%s)" % (
|
|
p.name, size, d.name, n.get_logicalname(d.name))
|
|
self.logger.debug(msg)
|
|
part = maas_dev.create_partition(part)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
|
|
if p.volume_group is not None:
|
|
self.logger.debug(
|
|
"Adding partition %s to volume group %s" %
|
|
(p.name, p.volume_group))
|
|
if p.volume_group not in vg_devs:
|
|
vg_devs[p.volume_group] = {'b': [], 'p': []}
|
|
vg_devs[p.volume_group]['p'].append(
|
|
part.resource_id)
|
|
|
|
if p.mountpoint is not None:
|
|
format_opts = {'fstype': p.fstype}
|
|
if p.fs_uuid is not None:
|
|
format_opts['uuid'] = str(p.fs_uuid)
|
|
if p.fs_label is not None:
|
|
format_opts['label'] = p.fs_label
|
|
|
|
msg = "Formatting partition %s as %s" % (p.name,
|
|
p.fstype)
|
|
self.logger.debug(msg)
|
|
part.format(**format_opts)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
mount_opts = {
|
|
'mount_point': p.mountpoint,
|
|
'mount_options': p.mount_options,
|
|
}
|
|
msg = "Mounting partition %s on %s" % (
|
|
p.name, p.mountpoint)
|
|
self.logger.debug(msg)
|
|
part.mount(**mount_opts)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
|
|
self.logger.debug(
|
|
"Finished configuring node %s partitions" % n.name)
|
|
|
|
for v in n.volume_groups:
|
|
if v.is_sys():
|
|
self.logger.debug(
|
|
"Skipping manually configuraing system VG.")
|
|
continue
|
|
if v.name not in vg_devs:
|
|
self.logger.warning(
|
|
"No physical volumes defined for VG %s, skipping."
|
|
% (v.name))
|
|
continue
|
|
|
|
maas_volgroup = maas_vg.VolumeGroup(
|
|
self.maas_client, name=v.name)
|
|
|
|
if v.vg_uuid is not None:
|
|
maas_volgroup.uuid = v.vg_uuid
|
|
|
|
if len(vg_devs[v.name]['b']) > 0:
|
|
maas_volgroup.block_devices = ','.join(
|
|
[str(x) for x in vg_devs[v.name]['b']])
|
|
if len(vg_devs[v.name]['p']) > 0:
|
|
maas_volgroup.partitions = ','.join(
|
|
[str(x) for x in vg_devs[v.name]['p']])
|
|
|
|
msg = "Create volume group %s on node %s" % (v.name,
|
|
n.name)
|
|
self.logger.debug(msg)
|
|
|
|
maas_volgroup = machine.volume_groups.add(maas_volgroup)
|
|
maas_volgroup.refresh()
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
|
|
for lv in v.logical_volumes:
|
|
calc_size = ApplyNodeStorage.calculate_bytes(
|
|
size_str=lv.size, context=maas_volgroup)
|
|
bd_id = maas_volgroup.create_lv(
|
|
name=lv.name, uuid_str=lv.lv_uuid, size=calc_size)
|
|
|
|
if lv.mountpoint is not None:
|
|
machine.refresh()
|
|
maas_lv = machine.block_devices.select(bd_id)
|
|
msg = "Formatting LV %s as filesystem on node %s." % (
|
|
lv.name, n.name)
|
|
self.logger.debug(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
maas_lv.format(
|
|
fstype=lv.fstype, uuid_str=lv.fs_uuid)
|
|
msg = "Mounting LV %s at %s on node %s." % (
|
|
lv.name, lv.mountpoint, n.name)
|
|
self.logger.debug(msg)
|
|
maas_lv.mount(
|
|
mount_point=lv.mountpoint,
|
|
mount_options=lv.mount_options)
|
|
self.task.add_status_msg(
|
|
msg=msg,
|
|
error=False,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
except Exception as ex:
|
|
self.task.failure(focus=n.get_id())
|
|
self.task.add_status_msg(
|
|
msg="Error configuring storage. %s" % str(ex),
|
|
error=True,
|
|
ctx=n.name,
|
|
ctx_type='node')
|
|
self.logger.debug("Error configuring storage for node %s: %s" %
|
|
(n.name, str(ex)))
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
|
|
return
|
|
|
|
@classmethod
|
|
def calculate_bytes(cls, size_str=None, context=None):
|
|
"""Calculate the size on bytes of a size_str.
|
|
|
|
Calculate the size as specified in size_str in the context of the provided
|
|
blockdev or vg. Valid size_str format below.
|
|
|
|
#m or #M or #mb or #MB = # * 1024 * 1024
|
|
#g or #G or #gb or #GB = # * 1024 * 1024 * 1024
|
|
#t or #T or #tb or #TB = # * 1024 * 1024 * 1024 * 1024
|
|
#% = Percentage of the total storage in the context
|
|
|
|
Prepend '>' to the above to note the size as a minimum and the calculated size being the
|
|
remaining storage available above the minimum
|
|
|
|
If the calculated size is not available in the context, a NotEnoughStorage exception is
|
|
raised.
|
|
|
|
:param size_str: A string representing the desired size
|
|
:param context: An instance of maasdriver.models.blockdev.BlockDevice or
|
|
instance of maasdriver.models.volumegroup.VolumeGroup. The
|
|
size_str is interpreted in the context of this device
|
|
:return size: The calculated size in bytes
|
|
"""
|
|
pattern = '(>?)(\d+)([mMbBgGtT%]{1,2})'
|
|
regex = re.compile(pattern)
|
|
match = regex.match(size_str)
|
|
|
|
if not match:
|
|
raise errors.InvalidSizeFormat(
|
|
"Invalid size string format: %s" % size_str)
|
|
|
|
if ((match.group(1) == '>' or match.group(3) == '%') and not context):
|
|
raise errors.InvalidSizeFormat(
|
|
'Sizes using the ">" or "%" format must specify a '
|
|
'block device or volume group context')
|
|
|
|
base_size = int(match.group(2))
|
|
|
|
if match.group(3) in ['m', 'M', 'mb', 'MB']:
|
|
computed_size = base_size * (1000 * 1000)
|
|
elif match.group(3) in ['g', 'G', 'gb', 'GB']:
|
|
computed_size = base_size * (1000 * 1000 * 1000)
|
|
elif match.group(3) in ['t', 'T', 'tb', 'TB']:
|
|
computed_size = base_size * (1000 * 1000 * 1000 * 1000)
|
|
elif match.group(3) == '%':
|
|
computed_size = math.floor((base_size / 100) * int(context.size))
|
|
|
|
if computed_size > int(context.available_size):
|
|
raise errors.NotEnoughStorage()
|
|
|
|
if match.group(1) == '>':
|
|
computed_size = int(context.available_size
|
|
) - ApplyNodeStorage.PART_TABLE_RESERVATION
|
|
|
|
return computed_size
|
|
|
|
|
|
class DeployNode(BaseMaasAction):
|
|
"""Action to write persistent OS to node."""
|
|
|
|
def start(self):
|
|
try:
|
|
machine_list = maas_machine.Machines(self.maas_client)
|
|
machine_list.refresh()
|
|
except Exception as ex:
|
|
self.logger.debug("Error accessing the MaaS API.", exc_info=ex)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg="Error accessing MaaS API: %s" % str(ex),
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.save()
|
|
return
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
site_design = self._load_site_design()
|
|
except errors.OrchestratorError:
|
|
self.task.add_status_msg(
|
|
msg="Error loading site design.",
|
|
error=True,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
nodes = self.orchestrator.process_node_filter(self.task.node_filter,
|
|
site_design)
|
|
|
|
for n in nodes:
|
|
try:
|
|
machine = find_node_in_maas(self.maas_client, n)
|
|
|
|
if type(machine) is maas_rack.RackController:
|
|
msg = "Skipping configuration of rack controller %s." % n.name
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
elif machine.status_name.startswith(
|
|
'Deployed') or machine.status_name.startswith(
|
|
'Deploying'):
|
|
msg = "Node %s already deployed or deploying, skipping." % (
|
|
n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.name)
|
|
continue
|
|
elif machine.status_name == 'Ready':
|
|
msg = "Acquiring node %s for deployment" % (n.name)
|
|
self.logger.info(msg)
|
|
machine = machine_list.acquire_node(n.name)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
else:
|
|
msg = "Unexpected status %s for node %s, skipping deployment." % (
|
|
machine.status_name, n.name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
except errors.DriverError as dex:
|
|
msg = "Error acquiring node %s, skipping" % n.name
|
|
self.logger.warning(msg, exc_info=dex)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
# Set owner data in MaaS
|
|
try:
|
|
self.logger.info("Setting node %s owner data." % n.name)
|
|
for k, v in n.owner_data.items():
|
|
msg = "Set owner data %s = %s for node %s" % (k, v, n.name)
|
|
self.logger.debug(msg)
|
|
machine.set_owner_data(k, v)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
except Exception as ex:
|
|
msg = "Error setting node %s owner data" % n.name
|
|
self.logger.warning(msg + ": " + str(ex))
|
|
self.task.failure(focus=n.get_id())
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
continue
|
|
|
|
# Saving boot action context for a node
|
|
self.logger.info(
|
|
"Saving Boot Action context for node %s." % (n.name))
|
|
try:
|
|
ba_key = self.orchestrator.create_bootaction_context(
|
|
n.name, self.task)
|
|
|
|
tag_list = maas_tag.Tags(self.maas_client)
|
|
tag_list.refresh()
|
|
node_id_tags = tag_list.startswith("%s__baid__" % (n.name))
|
|
for t in node_id_tags:
|
|
t.delete()
|
|
|
|
if ba_key is not None:
|
|
msg = "Creating boot action id key tag for node %s" % (
|
|
n.name)
|
|
self.logger.debug(msg)
|
|
node_baid_tag = maas_tag.Tag(
|
|
self.maas_client,
|
|
name="%s__baid__%s" % (n.name, ba_key.hex()))
|
|
node_baid_tag = tag_list.add(node_baid_tag)
|
|
node_baid_tag.apply_to_node(machine.resource_id)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
except Exception as ex:
|
|
self.logger.error(
|
|
"Error setting boot action id key tag for %s." % n.name,
|
|
exc_info=ex)
|
|
|
|
# Extract bootaction assets that are package lists as they
|
|
# are included in the deployment initiation
|
|
|
|
node_packages = self.orchestrator.find_node_package_lists(
|
|
n.name, self.task)
|
|
user_data_dict = dict(packages=[])
|
|
|
|
for k, v in node_packages.items():
|
|
if v:
|
|
user_data_dict['packages'].append([k, v])
|
|
else:
|
|
user_data_dict['packages'].append(k)
|
|
|
|
user_data_string = None
|
|
if user_data_dict.get('packages'):
|
|
user_data_string = "#cloud-config\n%s" % yaml.dump(
|
|
user_data_dict)
|
|
|
|
self.logger.info("Deploying node %s: image=%s, kernel=%s" %
|
|
(n.name, n.image, n.kernel))
|
|
|
|
try:
|
|
machine.deploy(
|
|
platform=n.image,
|
|
kernel=n.kernel,
|
|
user_data=user_data_string)
|
|
except errors.DriverError:
|
|
msg = "Error deploying node %s, skipping" % n.name
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
continue
|
|
|
|
attempts = 0
|
|
max_attempts = (
|
|
config.config_mgr.conf.timeouts.deploy_node *
|
|
60) // config.config_mgr.conf.maasdriver.poll_interval
|
|
|
|
while (attempts < max_attempts
|
|
and (not machine.status_name.startswith('Deployed')
|
|
and not machine.status_name.startswith('Failed'))):
|
|
attempts = attempts + 1
|
|
time.sleep(config.config_mgr.conf.maasdriver.poll_interval)
|
|
try:
|
|
machine.refresh()
|
|
self.logger.debug(
|
|
"Polling node %s status attempt %d of %d: %s" %
|
|
(n.name, attempts, max_attempts, machine.status_name))
|
|
except Exception:
|
|
self.logger.warning(
|
|
"Error updating node %s status during commissioning, will re-attempt."
|
|
% (n.name))
|
|
if machine.status_name.startswith('Deployed'):
|
|
msg = "Node %s deployed" % (n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=False, ctx=n.name, ctx_type='node')
|
|
self.task.success(focus=n.get_id())
|
|
elif machine.status_name.startswith('Failed'):
|
|
msg = "Node %s deployment failed" % (n.name)
|
|
self.logger.info(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
else:
|
|
msg = "Node %s deployment timed out" % (n.name)
|
|
self.logger.warning(msg)
|
|
self.task.add_status_msg(
|
|
msg=msg, error=True, ctx=n.name, ctx_type='node')
|
|
self.task.failure(focus=n.get_id())
|
|
self._add_detail_logs(n, machine, 'deploy', result_type='deploy')
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
|
|
return
|
|
|
|
def find_node_in_maas(maas_client, node_model):
|
|
"""Find a node in MAAS matching the node_model.
|
|
|
|
Note that the returned Machine may be a simple Machine or
|
|
a RackController.
|
|
|
|
:param maas_client: instance of an active session to MAAS
|
|
:param node_model: instance of objects.Node to match
|
|
:returns: instance of maasdriver.models.Machine
|
|
"""
|
|
|
|
machine_list = maas_machine.Machines(maas_client)
|
|
machine_list.refresh()
|
|
machine = machine_list.identify_baremetal_node(node_model)
|
|
|
|
if not machine:
|
|
# If node isn't found a normal node, check rack controllers
|
|
rackd_list = maas_rack.RackControllers(maas_client)
|
|
rackd_list.refresh()
|
|
machine = rackd_list.identify_baremetal_node(node_model)
|
|
|
|
return machine
|