231 lines
8.7 KiB
Python
231 lines
8.7 KiB
Python
# Copyright 2015 Red Hat, Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import fnmatch
|
|
import logging
|
|
import re
|
|
import time
|
|
|
|
import six
|
|
|
|
import heatclient.exc
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class DeployedServer(object):
|
|
id = None
|
|
name = None
|
|
|
|
|
|
class StackUpdateManager(object):
|
|
def __init__(self, heatclient, novaclient, stack, hook_type,
|
|
nested_depth=5, hook_resource=None):
|
|
self.heatclient = heatclient
|
|
self.novaclient = novaclient
|
|
self.stack = stack
|
|
self.hook_type = hook_type
|
|
self.nested_depth = nested_depth
|
|
self.hook_resource = hook_resource
|
|
self.server_names = {}
|
|
self.servers = []
|
|
|
|
def clear_breakpoints(self, refs):
|
|
resources = self._resources_by_state()
|
|
succeeds = []
|
|
fails = []
|
|
for ref in refs:
|
|
server_name = None
|
|
try:
|
|
res = resources['on_breakpoint'][ref]
|
|
server_name = self._server_name(ref)
|
|
LOG.info("removing breakpoint on %s", server_name)
|
|
stack_id = next(x['href'] for x in res.links if
|
|
x['rel'] == 'stack').rsplit('/', 1)[1]
|
|
self.heatclient.resources.signal(
|
|
stack_id=stack_id,
|
|
resource_name=res.logical_resource_id,
|
|
data={'unset_hook': self.hook_type})
|
|
succeeds.append(ref)
|
|
except Exception as err:
|
|
LOG.error("failed to remove breakpoint on %s: %s",
|
|
server_name or ref, err)
|
|
fails.append(ref)
|
|
return (succeeds, fails)
|
|
|
|
def get_status(self):
|
|
self.stack = self.heatclient.stacks.get(self.stack.id)
|
|
# check if any of deployments' child resource has last
|
|
# event indicating that it has reached a breakpoint (this
|
|
# seems to be the only way how to check pre-create breakpoints ATM)
|
|
resources = self._resources_by_state()
|
|
if self.stack.status == 'IN_PROGRESS':
|
|
if resources['on_breakpoint']:
|
|
if resources['in_progress']:
|
|
status = 'IN_PROGRESS'
|
|
else:
|
|
status = 'WAITING'
|
|
else:
|
|
status = 'IN_PROGRESS'
|
|
else:
|
|
status = self.stack.status
|
|
LOG.debug('%s status: %s', self.stack.stack_name, status)
|
|
return (status, resources)
|
|
|
|
def do_interactive_update(self):
|
|
status, _ = self.get_status()
|
|
|
|
# wait for the stack-update to start
|
|
while status in ['COMPLETE', 'FAILED']:
|
|
status, _ = self.get_status()
|
|
time.sleep(5)
|
|
|
|
while status not in ['COMPLETE', 'FAILED']:
|
|
status, resources = self.get_status()
|
|
print(status)
|
|
if status == 'WAITING':
|
|
for state in resources:
|
|
if resources[state]:
|
|
print("{0}: {1}".format(state, self._server_names(
|
|
resources[state].keys())))
|
|
user_input = six.moves.input(
|
|
"Breakpoint reached, continue? Regexp or "
|
|
"Enter=proceed (will clear %s), "
|
|
"C-c=quit interactive mode: "
|
|
% resources['on_breakpoint'].keys()[-1])
|
|
refs = self._input_to_refs(
|
|
user_input.strip(),
|
|
resources['on_breakpoint'].keys())
|
|
self.clear_breakpoints(refs)
|
|
time.sleep(5)
|
|
print('update finished with status {0}'.format(status))
|
|
|
|
def _resources_by_state(self):
|
|
resources = {
|
|
'not_started': {},
|
|
'in_progress': {},
|
|
'on_breakpoint': {},
|
|
'completed': {},
|
|
'failed': {},
|
|
}
|
|
all_resources = self.heatclient.resources.list(
|
|
self.stack.id, nested_depth=self.nested_depth)
|
|
if self.hook_type == 'pre-create':
|
|
hook_reason = 'CREATE paused until Hook pre-create is cleared'
|
|
hook_clear_reason = 'Hook pre-create is cleared'
|
|
else:
|
|
hook_reason = 'UPDATE paused until Hook pre-update is cleared'
|
|
hook_clear_reason = 'Hook pre-update is cleared'
|
|
|
|
stack_change_time = self._stack_change_time()
|
|
|
|
for res in all_resources:
|
|
if self.hook_resource:
|
|
if not fnmatch.fnmatchcase(res.resource_name,
|
|
self.hook_resource):
|
|
continue
|
|
stack_name, stack_id = next(
|
|
x['href'] for x in res.links if
|
|
x['rel'] == 'stack').rsplit('/', 2)[1:]
|
|
try:
|
|
events = self.heatclient.events.list(
|
|
stack_id=stack_id,
|
|
resource_name=res.logical_resource_id,
|
|
sort_dir='asc')
|
|
except heatclient.exc.HTTPNotFound:
|
|
events = []
|
|
state = 'not_started'
|
|
for ev in events:
|
|
# ignore events older than start of the last stack change
|
|
if ev.event_time < stack_change_time:
|
|
continue
|
|
if ev.resource_status_reason == hook_reason:
|
|
state = 'on_breakpoint'
|
|
elif ev.resource_status_reason == hook_clear_reason:
|
|
state = 'in_progress'
|
|
elif ev.resource_status in ('CREATE_IN_PROGRESS',
|
|
'UPDATE_IN_PROGRESS'):
|
|
state = 'in_progress'
|
|
elif ev.resource_status in ('CREATE_COMPLETE',
|
|
'UPDATE_COMPLETE'):
|
|
state = 'completed'
|
|
resources[state][res.physical_resource_id] = res
|
|
|
|
return resources
|
|
|
|
def _stack_change_time(self):
|
|
if self.hook_type == 'pre-create':
|
|
status_reason = 'Stack CREATE started'
|
|
else:
|
|
status_reason = 'Stack UPDATE started'
|
|
events = self.heatclient.events.list(
|
|
stack_id=self.stack.id,
|
|
sort_dir='desc')
|
|
try:
|
|
ev = next(e for e in events if
|
|
e.resource_status_reason == status_reason)
|
|
return ev.event_time
|
|
except StopIteration:
|
|
return None
|
|
|
|
def _server_names(self, deployment_ids):
|
|
return [self._server_name(i) for i in deployment_ids]
|
|
|
|
def _server_name(self, deployment_id):
|
|
name = self.server_names.get(deployment_id)
|
|
if not name:
|
|
if not self.servers:
|
|
self.servers = self._get_servers()
|
|
depl = self.heatclient.software_deployments.get(deployment_id)
|
|
name = next(server.name for server in self.servers if
|
|
server.id == depl.server_id)
|
|
self.server_names[deployment_id] = name
|
|
return name
|
|
|
|
def _get_servers(self):
|
|
servers = self.novaclient.servers.list()
|
|
|
|
# If no servers were found from Nova, we must be using split-stack,
|
|
# so we will have to interrogate Heat for the names and id's.
|
|
if not servers:
|
|
resources = self.heatclient.resources.list(
|
|
self.stack.id, nested_depth=self.nested_depth,
|
|
filters=dict(type="OS::Heat::DeployedServer"))
|
|
for res in resources:
|
|
server = DeployedServer()
|
|
stack_name, stack_id = next(
|
|
x['href'] for x in res.links if
|
|
x['rel'] == 'stack').rsplit('/', 2)[1:]
|
|
stack = self.heatclient.stacks.get(stack_id)
|
|
server.name = next(o['output_value'] for o in stack.outputs if
|
|
o['output_key'] == 'name')
|
|
server.id = res.physical_resource_id
|
|
servers.append(server)
|
|
|
|
return servers
|
|
|
|
def _input_to_refs(self, regexp, refs):
|
|
if regexp:
|
|
try:
|
|
pattern = "\A{0}\Z".format(regexp)
|
|
return [ref for ref in refs if
|
|
re.match(pattern, self._server_name(ref))]
|
|
except re.error as err:
|
|
LOG.warning("'%s' is invalid regular expression: %s",
|
|
regexp.encode('string-escape'), err)
|
|
return []
|
|
else:
|
|
return [refs.pop()]
|