1184 lines
45 KiB
Python
1184 lines
45 KiB
Python
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Actions for the Orchestrator level of the Drydock workflow."""
|
|
|
|
import time
|
|
import datetime
|
|
import logging
|
|
import concurrent.futures
|
|
import uuid
|
|
|
|
import drydock_provisioner.config as config
|
|
import drydock_provisioner.error as errors
|
|
import drydock_provisioner.objects.fields as hd_fields
|
|
|
|
|
|
class BaseAction(object):
|
|
"""The base class for actions starts by the orchestrator."""
|
|
|
|
def __init__(self, task, orchestrator, state_manager):
|
|
"""Object initializer.
|
|
|
|
:param task: objects.Task instance this action will execute against
|
|
:param orchestrator: orchestrator.Orchestrator instance
|
|
:param state_manager: state.DrydockState instnace used to access task state
|
|
"""
|
|
self.task = task
|
|
self.orchestrator = orchestrator
|
|
self.state_manager = state_manager
|
|
self.logger = logging.getLogger(
|
|
config.config_mgr.conf.logging.global_logger_name)
|
|
|
|
def _parallelize_subtasks(self, fn, subtask_id_list, *args, **kwargs):
|
|
"""Spawn threads to execute fn for each subtask using concurrent.futures.
|
|
|
|
Return a dictionary of task_id:concurrent.futures.Future instance
|
|
|
|
:param fn: The callable to execute in a thread, expected it takes a task_id as first argument
|
|
:param subtask_id_list: List of uuid.UUID ID of the subtasks to execute on
|
|
:param *args: The args to pass to fn
|
|
:param **kwargs: The kwargs to pass to fn
|
|
"""
|
|
task_futures = dict()
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as te:
|
|
for t in subtask_id_list:
|
|
task_futures[t.bytes] = te.submit(fn, t, *args, **kwargs)
|
|
|
|
return task_futures
|
|
|
|
def _split_action(self):
|
|
"""Start a parallel action for each node in scope.
|
|
|
|
Start a threaded instance of this action for each node in
|
|
scope of this action's task. A subtask will be created for each
|
|
action. Returns all a list of concurrent.futures managed
|
|
threads running the actions.
|
|
|
|
:returns: dictionary of subtask_id.bytes => Future instance
|
|
"""
|
|
target_nodes = self.orchestrator.get_target_nodes(self.task)
|
|
|
|
if len(target_nodes) > 1:
|
|
self.logger.info(
|
|
"Found multiple target nodes in task %s, splitting..." % str(
|
|
self.task.get_id()))
|
|
split_tasks = dict()
|
|
|
|
with concurrent.futures.ThreadPoolExecutor() as te:
|
|
for n in target_nodes:
|
|
split_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.PrepareNodes,
|
|
node_filter=self.orchestrator.
|
|
create_nodefilter_from_nodelist([n]))
|
|
self.task.register_subtask(split_task)
|
|
action = self.__class__(split_task, self.orchestrator,
|
|
self.state_manager)
|
|
split_tasks[split_task.get_id().bytes] = te.submit(
|
|
action.start)
|
|
|
|
return split_tasks
|
|
|
|
def _collect_subtask_futures(self, subtask_futures, timeout=300):
|
|
"""Collect Futures executing on subtasks or timeout.
|
|
|
|
Wait for Futures to finish or timeout. After timeout, enumerate the subtasks
|
|
that timed out in task result messages.
|
|
|
|
:param subtask_futures: dictionary of subtask_id.bytes -> Future instance
|
|
:param timeout: The number of seconds to wait for all Futures to complete
|
|
:param bubble: Whether to bubble results from collected subtasks
|
|
"""
|
|
finished, timed_out = concurrent.futures.wait(
|
|
subtask_futures.values(), timeout=timeout)
|
|
|
|
for k, v in subtask_futures.items():
|
|
if not v.done():
|
|
self.task.add_status_msg(
|
|
"Subtask thread for %s still executing after timeout." %
|
|
str(uuid.UUID(bytes=k)),
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.failure()
|
|
else:
|
|
if v.exception():
|
|
self.logger.error(
|
|
"Uncaught excetion in subtask %s future:" % str(
|
|
uuid.UUID(bytes=k)),
|
|
exc_info=v.exception())
|
|
st = self.state_manager.get_task(uuid.UUID(bytes=k))
|
|
st.bubble_results()
|
|
st.align_result()
|
|
st.save()
|
|
if len(timed_out) > 0:
|
|
raise errors.CollectSubtaskTimeout(
|
|
"One or more subtask threads did not finish in %d seconds." %
|
|
timeout)
|
|
|
|
self.task.align_result()
|
|
return
|
|
|
|
def _load_site_design(self, resolve_aliases=False):
|
|
"""Load the site design from this action's task.
|
|
|
|
The design_ref in the task can be resolved to a set of design documents
|
|
that reflect the site design to be operated on. Load this design for use
|
|
by this action.
|
|
|
|
:param resolve_aliases: boolean on whether to resolve device aliases, defaults
|
|
to False
|
|
"""
|
|
design_status, site_design = self.orchestrator.get_effective_site(
|
|
self.task.design_ref, resolve_aliases=resolve_aliases)
|
|
|
|
if design_status is None or design_status.status == hd_fields.ActionResult.Failure:
|
|
raise errors.OrchestratorError("Site design failed load.")
|
|
|
|
return site_design
|
|
|
|
def _get_driver(self, driver_type, subtype=None):
|
|
"""Locate the correct driver instance based on type and subtype.
|
|
|
|
:param driver_type: The type of driver, 'oob', 'node', or 'network'
|
|
:param subtype: In cases where multiple drivers can be active, select
|
|
one based on subtype
|
|
"""
|
|
driver = None
|
|
if driver_type == 'oob':
|
|
if subtype is None:
|
|
return None
|
|
for d in self.orchestrator.enabled_drivers['oob']:
|
|
if d.oob_type_support(subtype):
|
|
driver = d
|
|
break
|
|
else:
|
|
driver = self.orchestrator.enabled_drivers[driver_type]
|
|
|
|
return driver
|
|
|
|
|
|
class Noop(BaseAction):
|
|
"""Dummy action to allow the full task completion flow without impacts."""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.logger.debug("Starting Noop Action.")
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
time.sleep(5)
|
|
self.task = self.state_manager.get_task(self.task.get_id())
|
|
if self.task.check_terminate():
|
|
self.logger.debug("Terminating action.")
|
|
self.task.set_status(hd_fields.TaskStatus.Terminated)
|
|
self.task.failure()
|
|
self.task.add_status_msg(
|
|
msg="Action terminated.", ctx_type='NA', ctx='NA', error=False)
|
|
else:
|
|
self.logger.debug("Marked task as successful.")
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
target_nodes = self.orchestrator.get_target_nodes(self.task)
|
|
for n in target_nodes:
|
|
self.task.success(focus=n.name)
|
|
self.task.add_status_msg(
|
|
msg="Noop action.", ctx_type='NA', ctx='NA', error=False)
|
|
self.task.save()
|
|
self.logger.debug("Saved task state.")
|
|
self.logger.debug("Finished Noop Action.")
|
|
return
|
|
|
|
|
|
class DestroyNodes(BaseAction):
|
|
"""Action to destroy nodes in prepartion for a redeploy."""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
node_driver = self._get_driver('node')
|
|
|
|
if node_driver is None:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.add_status_msg(
|
|
msg="No node driver enabled, ending task.",
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.result.set_message("No NodeDriver enabled.")
|
|
self.task.result.set_reason("Bad Configuration.")
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
target_nodes = self.orchestrator.get_target_nodes(self.task)
|
|
|
|
if not target_nodes:
|
|
self.task.add_status_msg(
|
|
msg="No nodes in scope, no work to to do.",
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
node_release_task = None
|
|
while True:
|
|
if node_release_task is None:
|
|
node_release_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.DestroyNode,
|
|
node_filter=self.task.node_filter)
|
|
self.task.register_subtask(node_release_task)
|
|
|
|
self.logger.info("Starting node driver task %s to Release nodes." %
|
|
(node_release_task.get_id()))
|
|
node_driver.execute_task(node_release_task.get_id())
|
|
|
|
node_release_task = self.state_manager.get_task(
|
|
node_release_task.get_id())
|
|
|
|
try:
|
|
if not node_release_task.retry_task(max_attempts=3):
|
|
break
|
|
except errors.MaxRetriesReached:
|
|
self.task.failure()
|
|
break
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.DestroyNode)
|
|
self.task.align_result()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class ValidateDesign(BaseAction):
|
|
"""Action for validating the design document referenced by the task."""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
try:
|
|
status, site_design = self.orchestrator.get_effective_site(
|
|
self.task.design_ref)
|
|
self.task.merge_status_messages(task_result=status)
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
if status.result.status == hd_fields.ActionResult.Success:
|
|
self.task.success()
|
|
else:
|
|
self.task.failure()
|
|
self.task.save()
|
|
except Exception:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class VerifySite(BaseAction):
|
|
"""Action to verify downstream tools in the site are available and ready."""
|
|
|
|
def start(self):
|
|
"""Start executing this action in the context of the local task."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
node_driver = self._get_driver('node')
|
|
|
|
if node_driver is None:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.add_status_msg(
|
|
msg="No node driver enabled, ending task.",
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.result.set_message("No NodeDriver enabled.")
|
|
self.task.result.set_reason("Bad Configuration.")
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
node_driver_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ValidateNodeServices)
|
|
self.task.register_subtask(node_driver_task)
|
|
|
|
node_driver.execute_task(node_driver_task.get_id())
|
|
|
|
node_driver_task = self.state_manager.get_task(
|
|
node_driver_task.get_id())
|
|
|
|
self.task.add_status_msg(
|
|
msg="Collected subtask %s" % str(node_driver_task.get_id()),
|
|
error=False,
|
|
ctx=str(node_driver_task.get_id()),
|
|
ctx_type='task')
|
|
|
|
self.task = self.state_manager.get_task(self.task.get_id())
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.align_result()
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class PrepareSite(BaseAction):
|
|
"""Action to configure site wide/inter-node settings."""
|
|
|
|
def start(self):
|
|
"""Start executing this action in the context of the local task."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
driver = self._get_driver('node')
|
|
|
|
if driver is None:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.add_status_msg(
|
|
msg="No node driver enabled, ending task.",
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.result.set_message("No NodeDriver enabled.")
|
|
self.task.result.set_reason("Bad Configuration.")
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
self.step_networktemplate(driver)
|
|
self.step_usercredentials(driver)
|
|
self.step_configureprovisioner(driver)
|
|
|
|
self.task.align_result()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
def step_configureprovisioner(self, driver):
|
|
"""Run the ConfigureNodeProvisioner step of this action.
|
|
|
|
:param driver: The driver instance to use for execution.
|
|
"""
|
|
config_prov_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ConfigureNodeProvisioner)
|
|
self.task.register_subtask(config_prov_task)
|
|
|
|
self.logger.info(
|
|
"Starting node drvier task %s to configure the provisioner" %
|
|
(config_prov_task.get_id()))
|
|
|
|
driver.execute_task(config_prov_task.get_id())
|
|
|
|
self.task.add_status_msg(
|
|
msg="Collected subtask %s" % str(config_prov_task.get_id()),
|
|
error=False,
|
|
ctx=str(config_prov_task.get_id()),
|
|
ctx_type='task')
|
|
self.logger.info("Node driver task %s:%s is complete." %
|
|
(config_prov_task.get_id(), config_prov_task.action))
|
|
|
|
def step_networktemplate(self, driver):
|
|
"""Run the CreateNetworkTemplate step of this action.
|
|
|
|
:param driver: The driver instance to use for execution.
|
|
"""
|
|
site_network_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.CreateNetworkTemplate)
|
|
self.task.register_subtask(site_network_task)
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to create network templates" %
|
|
(site_network_task.get_id()))
|
|
|
|
driver.execute_task(site_network_task.get_id())
|
|
|
|
self.task.add_status_msg(
|
|
msg="Collected subtask %s" % str(site_network_task.get_id()),
|
|
error=False,
|
|
ctx=str(site_network_task.get_id()),
|
|
ctx_type='task')
|
|
self.logger.info(
|
|
"Node driver task %s complete" % (site_network_task.get_id()))
|
|
|
|
def step_usercredentials(self, driver):
|
|
"""Run the ConfigureUserCredentials step of this action.
|
|
|
|
:param driver: The driver instance to use for execution.
|
|
"""
|
|
user_creds_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ConfigureUserCredentials)
|
|
self.task.register_subtask(user_creds_task)
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to configure user credentials" %
|
|
(user_creds_task.get_id()))
|
|
|
|
driver.execute_task(user_creds_task.get_id())
|
|
|
|
self.task.add_status_msg(
|
|
msg="Collected subtask %s" % str(user_creds_task.get_id()),
|
|
error=False,
|
|
ctx=str(user_creds_task.get_id()),
|
|
ctx_type='task')
|
|
self.logger.info(
|
|
"Node driver task %s complete" % (user_creds_task.get_id()))
|
|
|
|
|
|
class VerifyNodes(BaseAction):
|
|
"""Action to verify the orchestrator has adequate access to a node to start the deployment."""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
design_status, site_design = self.orchestrator.get_effective_site(
|
|
self.task.design_ref)
|
|
|
|
node_filter = self.task.node_filter
|
|
|
|
oob_type_partition = {}
|
|
|
|
target_nodes = self.orchestrator.process_node_filter(
|
|
node_filter, site_design)
|
|
|
|
if target_nodes:
|
|
for n in target_nodes:
|
|
if n.oob_type not in oob_type_partition.keys():
|
|
oob_type_partition[n.oob_type] = []
|
|
|
|
oob_type_partition[n.oob_type].append(n)
|
|
|
|
task_futures = dict()
|
|
for oob_type, oob_nodes in oob_type_partition.items():
|
|
oob_driver = self._get_driver('oob', oob_type)
|
|
|
|
if oob_driver is None:
|
|
self.logger.warning(
|
|
"Node OOB type %s has no enabled driver." % oob_type)
|
|
self.task.failure()
|
|
for n in oob_nodes:
|
|
self.task.add_status_msg(
|
|
msg="Node %s OOB type %s is not supported." %
|
|
(n.get_name(), oob_type),
|
|
error=True,
|
|
ctx=n.get_name(),
|
|
ctx_type='node')
|
|
continue
|
|
|
|
nf = self.orchestrator.create_nodefilter_from_nodelist(
|
|
oob_nodes)
|
|
|
|
oob_driver_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.InterrogateOob,
|
|
node_filter=nf)
|
|
self.task.register_subtask(oob_driver_task)
|
|
|
|
self.logger.info(
|
|
"Starting task %s for node verification via OOB type %s" %
|
|
(oob_driver_task.get_id(), oob_type))
|
|
task_futures.update(
|
|
self._parallelize_subtasks(oob_driver.execute_task,
|
|
[oob_driver_task.get_id()]))
|
|
|
|
try:
|
|
self._collect_subtask_futures(
|
|
task_futures,
|
|
timeout=(
|
|
config.config_mgr.conf.timeouts.drydock_timeout * 60))
|
|
self.logger.debug(
|
|
"Collected subtasks for task %s" % str(self.task.get_id()))
|
|
except errors.CollectSubtaskTimeout as ex:
|
|
self.logger.warning(str(ex))
|
|
else:
|
|
# no target nodes
|
|
self.task.add_status_msg(
|
|
msg="No nodes in scope, no work to to do.",
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
|
|
# Set task complete and persist that info.
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class PrepareNodes(BaseAction):
|
|
"""Action to prepare a node for deployment."""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
node_driver = self._get_driver('node')
|
|
|
|
if node_driver is None:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.add_status_msg(
|
|
msg="No node driver enabled, ending task.",
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.result.set_message("No NodeDriver enabled.")
|
|
self.task.result.set_reason("Bad Configuration.")
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
design_status, site_design = self.orchestrator.get_effective_site(
|
|
self.task.design_ref)
|
|
|
|
target_nodes = self.orchestrator.get_target_nodes(self.task)
|
|
|
|
if not target_nodes:
|
|
self.task.add_status_msg(
|
|
msg="No nodes in scope, no work to to do.",
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
# Parallelize this task so that node progression is
|
|
# not interlinked
|
|
if len(target_nodes) > 1:
|
|
self.split_action()
|
|
else:
|
|
check_task_id = self.step_checkexist(node_driver)
|
|
node_check_task = self.state_manager.get_task(check_task_id)
|
|
|
|
# Build the node_filter from the failures of the node_check_task
|
|
# as these are the nodes that are unknown to the node provisioner
|
|
# and are likely save to be rebooted
|
|
|
|
target_nodes = self.orchestrator.process_node_filter(
|
|
node_check_task.node_filter_from_failures(), site_design)
|
|
|
|
# And log the nodes that were found so they can be addressed
|
|
known_nodes = self.orchestrator.process_node_filter(
|
|
node_check_task.node_filter_from_successes(), site_design)
|
|
|
|
for n in node_check_task.result.successes:
|
|
self.logger.debug(
|
|
"Found node %s in provisioner, skipping OOB management." %
|
|
(n))
|
|
self.task.add_status_msg(
|
|
msg="Node found in provisioner, skipping OOB management",
|
|
error=False,
|
|
ctx=n,
|
|
ctx_type='node')
|
|
|
|
self.step_oob_set_netboot(target_nodes)
|
|
|
|
# bubble results from the SetNodeBoot task and
|
|
# continue the execution with successful targets
|
|
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.SetNodeBoot)
|
|
|
|
target_nodes = self.orchestrator.process_node_filter(
|
|
self.task.node_filter_from_successes(), site_design)
|
|
|
|
self.step_oob_powercycle(target_nodes)
|
|
|
|
# bubble results from the Powercycle task and
|
|
# continue the execution with successful targets
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.PowerCycleNode)
|
|
|
|
target_nodes = self.orchestrator.process_node_filter(
|
|
self.task.node_filter_from_successes(), site_design)
|
|
|
|
self.step_node_identify(node_driver, target_nodes)
|
|
|
|
# We can only commission nodes that were successfully identified in the provisioner
|
|
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.IdentifyNode)
|
|
|
|
target_nodes = self.orchestrator.process_node_filter(
|
|
self.task.node_filter_from_successes(), site_design)
|
|
|
|
target_nodes.extend(known_nodes)
|
|
self.step_node_configure_hw(node_driver, target_nodes)
|
|
|
|
self.task.bubble_results()
|
|
self.task.align_result()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
def step_node_configure_hw(self, node_driver, node_list):
|
|
"""Execute the ConfigureHardware step of this action on a list of nodes.
|
|
|
|
:param node_driver: driver instance to use for execution
|
|
:param node_list: a list of objects.BaremetalNode instances
|
|
"""
|
|
if len(node_list) > 0:
|
|
self.logger.info(
|
|
"Starting hardware configuration task on %d nodes." %
|
|
len(node_list))
|
|
|
|
node_commission_task = None
|
|
while True:
|
|
if node_commission_task is None:
|
|
node_commission_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ConfigureHardware,
|
|
node_filter=self.orchestrator.
|
|
create_nodefilter_from_nodelist(node_list))
|
|
self.task.register_subtask(node_commission_task)
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to commission nodes." %
|
|
(node_commission_task.get_id()))
|
|
|
|
node_driver.execute_task(node_commission_task.get_id())
|
|
|
|
node_commission_task = self.state_manager.get_task(
|
|
node_commission_task.get_id())
|
|
try:
|
|
if not node_commission_task.retry_task(max_attempts=3):
|
|
break
|
|
except errors.MaxRetriesReached:
|
|
self.task.failure()
|
|
break
|
|
else:
|
|
self.logger.warning(
|
|
"No nodes successfully identified, skipping commissioning subtask"
|
|
)
|
|
|
|
def step_node_identify(self, node_driver, node_list):
|
|
"""Execute the IdentifyNode step of this action on a list of nodes.
|
|
|
|
:param node_driver: driver instance to use for execution
|
|
:param node_list: a list of objects.BaremetalNode instances
|
|
:return: list of uuid.UUID task ids of the tasks executing this step
|
|
"""
|
|
# IdentifyNode success will take some time after PowerCycleNode finishes
|
|
# Retry the operation a few times if it fails before considering it a final failure
|
|
# Each attempt is a new task which might make the final task tree a bit confusing
|
|
|
|
max_attempts = config.config_mgr.conf.timeouts.identify_node * (
|
|
60 / config.config_mgr.conf.poll_interval)
|
|
|
|
self.logger.debug(
|
|
"Will make max of %d attempts to complete the identify_node task."
|
|
% max_attempts)
|
|
node_identify_task = None
|
|
|
|
while True:
|
|
if node_identify_task is None:
|
|
node_identify_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.IdentifyNode,
|
|
node_filter=self.orchestrator.
|
|
create_nodefilter_from_nodelist(node_list))
|
|
self.task.register_subtask(node_identify_task)
|
|
|
|
self.logger.info("Starting node driver task %s to identify nodes."
|
|
% (node_identify_task.get_id()))
|
|
|
|
node_driver.execute_task(node_identify_task.get_id())
|
|
|
|
node_identify_task = self.state_manager.get_task(
|
|
node_identify_task.get_id())
|
|
node_identify_task.bubble_results()
|
|
|
|
try:
|
|
if not node_identify_task.retry_task(
|
|
max_attempts=max_attempts):
|
|
break
|
|
|
|
time.sleep(config.config_mgr.conf.poll_interval)
|
|
except errors.MaxRetriesReached:
|
|
self.task.failure()
|
|
break
|
|
|
|
return [node_identify_task.get_id()]
|
|
|
|
def step_oob_set_netboot(self, node_list):
|
|
"""Execute the SetNetBoot step of this action on a list of nodes.
|
|
|
|
:param node_list: a list of objects.BaremetalNode instances
|
|
:return: list of uuid.UUID task ids of the tasks executing this step
|
|
"""
|
|
oob_type_partition = {}
|
|
|
|
for n in node_list:
|
|
if n.oob_type not in oob_type_partition.keys():
|
|
oob_type_partition[n.oob_type] = []
|
|
|
|
oob_type_partition[n.oob_type].append(n)
|
|
|
|
task_futures = dict()
|
|
|
|
for oob_type, oob_nodes in oob_type_partition.items():
|
|
oob_driver = self._get_driver('oob', oob_type)
|
|
|
|
if oob_driver is None:
|
|
self.logger.warning(
|
|
"Node OOB type %s has no enabled driver." % oob_type)
|
|
self.task.failure()
|
|
for n in oob_nodes:
|
|
self.task.add_status_msg(
|
|
msg="Node %s OOB type %s is not supported." %
|
|
(n.get_name(), oob_type),
|
|
error=True,
|
|
ctx=n.get_name(),
|
|
ctx_type='node')
|
|
continue
|
|
|
|
setboot_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.SetNodeBoot,
|
|
node_filter=self.orchestrator.create_nodefilter_from_nodelist(
|
|
oob_nodes))
|
|
self.task.register_subtask(setboot_task)
|
|
|
|
self.logger.info(
|
|
"Starting OOB driver task %s to set PXE boot for OOB type %s" %
|
|
(setboot_task.get_id(), oob_type))
|
|
task_futures.update(
|
|
self._parallelize_subtasks(oob_driver.execute_task,
|
|
[setboot_task.get_id()]))
|
|
|
|
try:
|
|
self._collect_subtask_futures(
|
|
task_futures,
|
|
timeout=(config.config_mgr.conf.timeouts.drydock_timeout * 60))
|
|
self.logger.debug(
|
|
"Collected subtasks for task %s" % str(self.task.get_id()))
|
|
except errors.CollectSubtaskTimeout as ex:
|
|
self.logger.warning(str(ex))
|
|
|
|
return [uuid.UUID(bytes=x) for x in task_futures.keys()]
|
|
|
|
def step_oob_powercycle(self, node_list):
|
|
"""Execute the NodePowerCycle step of this action on a list of nodes.
|
|
|
|
:param node_list: a list of objects.BaremetalNode instances
|
|
:return: list of uuid.UUID task ids of the tasks executing this step
|
|
"""
|
|
oob_type_partition = {}
|
|
|
|
for n in node_list:
|
|
if n.oob_type not in oob_type_partition.keys():
|
|
oob_type_partition[n.oob_type] = []
|
|
|
|
oob_type_partition[n.oob_type].append(n)
|
|
|
|
task_futures = dict()
|
|
|
|
for oob_type, oob_nodes in oob_type_partition.items():
|
|
oob_driver = self._get_driver('oob', oob_type)
|
|
|
|
if oob_driver is None:
|
|
self.logger.warning(
|
|
"Node OOB type %s has no enabled driver." % oob_type)
|
|
self.task.failure()
|
|
for n in oob_nodes:
|
|
self.task.add_status_msg(
|
|
msg="Node %s OOB type %s is not supported." %
|
|
(n.get_name(), oob_type),
|
|
error=True,
|
|
ctx=n.get_name(),
|
|
ctx_type='node')
|
|
continue
|
|
|
|
cycle_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.PowerCycleNode,
|
|
node_filter=self.orchestrator.create_nodefilter_from_nodelist(
|
|
oob_nodes))
|
|
self.task.register_subtask(cycle_task)
|
|
|
|
self.logger.info(
|
|
"Starting OOB driver task %s to power cycle nodes for OOB type %s"
|
|
% (cycle_task.get_id(), oob_type))
|
|
|
|
task_futures.update(
|
|
self._parallelize_subtasks(oob_driver.execute_task,
|
|
[cycle_task.get_id()]))
|
|
|
|
try:
|
|
self._collect_subtask_futures(
|
|
task_futures,
|
|
timeout=(config.config_mgr.conf.timeouts.drydock_timeout * 60))
|
|
self.logger.debug(
|
|
"Collected subtasks for task %s" % str(self.task.get_id()))
|
|
except errors.CollectSubtaskTimeout as ex:
|
|
self.logger.warning(str(ex))
|
|
|
|
return [uuid.UUID(bytes=x) for x in task_futures.keys()]
|
|
|
|
def split_action(self):
|
|
"""Split this action into independent threads per node."""
|
|
action_timeout = config.config_mgr.conf.timeouts.identify_node + \
|
|
config.config_mgr.conf.timeouts.configure_hardware
|
|
split_tasks = self._split_action()
|
|
self._collect_subtask_futures(split_tasks, timeout=action_timeout * 60)
|
|
|
|
def step_checkexist(self, driver):
|
|
"""Query the driver for node existence to prevent impacting deployed nodes.
|
|
|
|
:param driver: driver instance to use for execution
|
|
:returns: uuid.UUID task id of the task that executed this step
|
|
"""
|
|
node_check_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.IdentifyNode,
|
|
node_filter=self.task.node_filter)
|
|
|
|
self.logger.info("Starting check task %s before rebooting nodes" %
|
|
(node_check_task.get_id()))
|
|
|
|
task_futures = self._parallelize_subtasks(driver.execute_task,
|
|
[node_check_task.get_id()])
|
|
self._collect_subtask_futures(
|
|
task_futures,
|
|
timeout=(config.config_mgr.conf.timeouts.drydock_timeout * 60))
|
|
|
|
node_check_task = self.state_manager.get_task(node_check_task.get_id())
|
|
node_check_task.bubble_results()
|
|
node_check_task.save()
|
|
|
|
return node_check_task.get_id()
|
|
|
|
|
|
class DeployNodes(BaseAction):
|
|
"""Action to deploy a node with a persistent OS."""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
node_driver = self.orchestrator.enabled_drivers['node']
|
|
|
|
if node_driver is None:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.add_status_msg(
|
|
msg="No node driver enabled, ending task.",
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.result.set_message("No NodeDriver enabled.")
|
|
self.task.result.set_reason("Bad Configuration.")
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
target_nodes = self.orchestrator.get_target_nodes(self.task)
|
|
|
|
if not target_nodes:
|
|
self.task.add_status_msg(
|
|
msg="No nodes in scope, no work to to do.",
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
node_networking_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ApplyNodeNetworking,
|
|
node_filter=self.task.node_filter)
|
|
self.task.register_subtask(node_networking_task)
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to apply networking on nodes." %
|
|
(node_networking_task.get_id()))
|
|
|
|
node_driver.execute_task(node_networking_task.get_id())
|
|
|
|
node_networking_task = self.state_manager.get_task(
|
|
node_networking_task.get_id())
|
|
|
|
node_storage_task = None
|
|
|
|
if len(node_networking_task.result.successes) > 0:
|
|
self.logger.info(
|
|
"Found %s successfully networked nodes, configuring storage." %
|
|
(len(node_networking_task.result.successes)))
|
|
|
|
node_storage_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ApplyNodeStorage,
|
|
node_filter=node_networking_task.node_filter_from_successes())
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to configure node storage." %
|
|
(node_storage_task.get_id()))
|
|
|
|
node_driver.execute_task(node_storage_task.get_id())
|
|
|
|
node_storage_task = self.state_manager.get_task(
|
|
node_storage_task.get_id())
|
|
|
|
else:
|
|
self.logger.warning(
|
|
"No nodes successfully networked, skipping storage configuration subtask."
|
|
)
|
|
|
|
node_platform_task = None
|
|
if (node_storage_task is not None
|
|
and len(node_storage_task.result.successes) > 0):
|
|
self.logger.info(
|
|
"Configured storage on %s nodes, configuring platform." % (len(
|
|
node_storage_task.result.successes)))
|
|
|
|
node_platform_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.ApplyNodePlatform,
|
|
node_filter=node_storage_task.node_filter_from_successes())
|
|
self.task.register_subtask(node_platform_task)
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to configure node platform." %
|
|
(node_platform_task.get_id()))
|
|
|
|
node_driver.execute_task(node_platform_task.get_id())
|
|
|
|
node_platform_task = self.state_manager.get_task(
|
|
node_platform_task.get_id())
|
|
|
|
else:
|
|
self.logger.warning(
|
|
"No nodes with storage configuration, skipping platform configuration subtask."
|
|
)
|
|
|
|
node_deploy_task = None
|
|
if node_platform_task is not None and len(
|
|
node_platform_task.result.successes) > 0:
|
|
self.logger.info(
|
|
"Configured platform on %s nodes, starting deployment." % (len(
|
|
node_platform_task.result.successes)))
|
|
|
|
while True:
|
|
if node_deploy_task is None:
|
|
node_deploy_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.DeployNode,
|
|
node_filter=node_platform_task.
|
|
node_filter_from_successes())
|
|
self.task.register_subtask(node_deploy_task)
|
|
|
|
self.logger.info(
|
|
"Starting node driver task %s to deploy nodes." %
|
|
(node_deploy_task.get_id()))
|
|
node_driver.execute_task(node_deploy_task.get_id())
|
|
|
|
node_deploy_task = self.state_manager.get_task(
|
|
node_deploy_task.get_id())
|
|
|
|
try:
|
|
if not node_deploy_task.retry_task(max_attempts=3):
|
|
break
|
|
except errors.MaxRetriesReached:
|
|
self.task.failure()
|
|
break
|
|
else:
|
|
self.logger.warning(
|
|
"Unable to configure platform on any nodes, skipping deploy subtask"
|
|
)
|
|
|
|
if (node_deploy_task is not None
|
|
and node_deploy_task.result is not None
|
|
and len(node_deploy_task.result.successes) > 0):
|
|
node_bootaction_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.BootactionReport,
|
|
node_filter=node_deploy_task.node_filter_from_successes())
|
|
self.task.register_subtask(node_bootaction_task)
|
|
action = BootactionReport(node_bootaction_task, self.orchestrator,
|
|
self.state_manager)
|
|
action.start()
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.BootactionReport)
|
|
self.task.align_result()
|
|
else:
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.DeployNode)
|
|
self.task.align_result()
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
|
|
class RelabelNodes(BaseAction):
|
|
"""Action to relabel a node"""
|
|
|
|
def start(self):
|
|
"""Start executing this action."""
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
kubernetes_driver = self.orchestrator.enabled_drivers['kubernetes']
|
|
|
|
if kubernetes_driver is None:
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.add_status_msg(
|
|
msg="No kubernetes driver is enabled, ending task.",
|
|
error=True,
|
|
ctx=str(self.task.get_id()),
|
|
ctx_type='task')
|
|
self.task.result.set_message("No KubernetesDriver enabled.")
|
|
self.task.result.set_reason("Bad Configuration.")
|
|
self.task.failure()
|
|
self.task.save()
|
|
return
|
|
|
|
target_nodes = self.orchestrator.get_target_nodes(self.task)
|
|
|
|
if not target_nodes:
|
|
self.task.add_status_msg(
|
|
msg="No nodes in scope, nothing to relabel.",
|
|
error=False,
|
|
ctx='NA',
|
|
ctx_type='NA')
|
|
self.task.success()
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|
|
|
|
nf = self.orchestrator.create_nodefilter_from_nodelist(target_nodes)
|
|
|
|
relabel_node_task = self.orchestrator.create_task(
|
|
design_ref=self.task.design_ref,
|
|
action=hd_fields.OrchestratorAction.RelabelNode,
|
|
node_filter=nf)
|
|
self.task.register_subtask(relabel_node_task)
|
|
|
|
self.logger.info("Starting kubernetes driver task %s to relabel nodes."
|
|
% (relabel_node_task.get_id()))
|
|
kubernetes_driver.execute_task(relabel_node_task.get_id())
|
|
|
|
relabel_node_task = self.state_manager.get_task(
|
|
relabel_node_task.get_id())
|
|
|
|
self.task.bubble_results(
|
|
action_filter=hd_fields.OrchestratorAction.RelabelNode)
|
|
self.task.align_result()
|
|
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
|
|
|
|
class BootactionReport(BaseAction):
|
|
"""Wait for nodes to report status of boot action."""
|
|
|
|
def start(self):
|
|
self.task.set_status(hd_fields.TaskStatus.Running)
|
|
self.task.save()
|
|
|
|
poll_start = datetime.datetime.utcnow()
|
|
|
|
still_running = True
|
|
timeout = datetime.timedelta(
|
|
minutes=config.config_mgr.conf.timeouts.bootaction_final_status)
|
|
running_time = datetime.datetime.utcnow() - poll_start
|
|
nodelist = [
|
|
n.get_id() for n in self.orchestrator.get_target_nodes(self.task)
|
|
]
|
|
|
|
self.logger.debug(
|
|
"Waiting for bootaction response signals to complete.")
|
|
while running_time < timeout:
|
|
still_running = False
|
|
for n in nodelist:
|
|
bas = self.state_manager.get_boot_actions_for_node(n)
|
|
running_bas = {
|
|
k: v
|
|
for (k, v) in bas.items() if v.get('action_status') ==
|
|
hd_fields.ActionResult.Incomplete
|
|
}
|
|
if len(running_bas) > 0:
|
|
still_running = True
|
|
break
|
|
if still_running:
|
|
self.logger.debug("Still waiting on %d running bootactions." %
|
|
len(running_bas))
|
|
time.sleep(config.config_mgr.conf.poll_interval)
|
|
running_time = datetime.datetime.utcnow() - poll_start
|
|
else:
|
|
break
|
|
|
|
self.logger.debug("Signals complete or timeout reached.")
|
|
|
|
for n in nodelist:
|
|
bas = self.state_manager.get_boot_actions_for_node(n)
|
|
success_bas = {
|
|
k: v
|
|
for (k, v) in bas.items()
|
|
if v.get('action_status') == hd_fields.ActionResult.Success
|
|
}
|
|
running_bas = {
|
|
k: v
|
|
for (k, v) in bas.items()
|
|
if v.get('action_status') == hd_fields.ActionResult.Incomplete
|
|
}
|
|
failure_bas = {
|
|
k: v
|
|
for (k, v) in bas.items()
|
|
if v.get('action_status') == hd_fields.ActionResult.Failure
|
|
}
|
|
for ba in success_bas.values():
|
|
self.task.add_status_msg(
|
|
msg="Boot action %s completed with status %s" %
|
|
(ba['action_name'], ba['action_status']),
|
|
error=False,
|
|
ctx=n,
|
|
ctx_type='node')
|
|
for ba in failure_bas.values():
|
|
self.task.add_status_msg(
|
|
msg="Boot action %s completed with status %s" %
|
|
(ba['action_name'], ba['action_status']),
|
|
error=True,
|
|
ctx=n,
|
|
ctx_type='node')
|
|
for ba in running_bas.values():
|
|
self.task.add_status_msg(
|
|
msg="Boot action %s timed out." % (ba['action_name']),
|
|
error=True,
|
|
ctx=n,
|
|
ctx_type='node')
|
|
|
|
if len(failure_bas) == 0 and len(running_bas) == 0:
|
|
self.task.success(focus=n)
|
|
else:
|
|
self.task.failure(focus=n)
|
|
|
|
self.logger.debug("Completed collecting bootaction signals.")
|
|
self.task.set_status(hd_fields.TaskStatus.Complete)
|
|
self.task.save()
|
|
return
|