39c8616876
On a minor interactive update, we never prompoted to clear breakpoints when using deployed-server since the code reads the server id's from nova, of which there are none. This modifies the behavior to read the server id's and names from Heat when nova returns no servers. Change-Id: I682f6dc66705c9d42b9c2d21f675491ea60c9c3c Closes-Bug: #1708236
231 lines
8.7 KiB
Python
231 lines
8.7 KiB
Python
# Copyright 2015 Red Hat, Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import fnmatch
|
|
import logging
|
|
import re
|
|
import time
|
|
|
|
import six
|
|
|
|
import heatclient.exc
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class DeployedServer(object):
|
|
id = None
|
|
name = None
|
|
|
|
|
|
class StackUpdateManager(object):
|
|
def __init__(self, heatclient, novaclient, stack, hook_type,
|
|
nested_depth=5, hook_resource=None):
|
|
self.heatclient = heatclient
|
|
self.novaclient = novaclient
|
|
self.stack = stack
|
|
self.hook_type = hook_type
|
|
self.nested_depth = nested_depth
|
|
self.hook_resource = hook_resource
|
|
self.server_names = {}
|
|
self.servers = []
|
|
|
|
def clear_breakpoints(self, refs):
|
|
resources = self._resources_by_state()
|
|
succeeds = []
|
|
fails = []
|
|
for ref in refs:
|
|
server_name = None
|
|
try:
|
|
res = resources['on_breakpoint'][ref]
|
|
server_name = self._server_name(ref)
|
|
LOG.info("removing breakpoint on %s", server_name)
|
|
stack_id = next(x['href'] for x in res.links if
|
|
x['rel'] == 'stack').rsplit('/', 1)[1]
|
|
self.heatclient.resources.signal(
|
|
stack_id=stack_id,
|
|
resource_name=res.logical_resource_id,
|
|
data={'unset_hook': self.hook_type})
|
|
succeeds.append(ref)
|
|
except Exception as err:
|
|
LOG.error("failed to remove breakpoint on %s: %s",
|
|
server_name or ref, err)
|
|
fails.append(ref)
|
|
return (succeeds, fails)
|
|
|
|
def get_status(self):
|
|
self.stack = self.heatclient.stacks.get(self.stack.id)
|
|
# check if any of deployments' child resource has last
|
|
# event indicating that it has reached a breakpoint (this
|
|
# seems to be the only way how to check pre-create breakpoints ATM)
|
|
resources = self._resources_by_state()
|
|
if self.stack.status == 'IN_PROGRESS':
|
|
if resources['on_breakpoint']:
|
|
if resources['in_progress']:
|
|
status = 'IN_PROGRESS'
|
|
else:
|
|
status = 'WAITING'
|
|
else:
|
|
status = 'IN_PROGRESS'
|
|
else:
|
|
status = self.stack.status
|
|
LOG.debug('%s status: %s', self.stack.stack_name, status)
|
|
return (status, resources)
|
|
|
|
def do_interactive_update(self):
|
|
status, _ = self.get_status()
|
|
|
|
# wait for the stack-update to start
|
|
while status in ['COMPLETE', 'FAILED']:
|
|
status, _ = self.get_status()
|
|
time.sleep(5)
|
|
|
|
while status not in ['COMPLETE', 'FAILED']:
|
|
status, resources = self.get_status()
|
|
print(status)
|
|
if status == 'WAITING':
|
|
for state in resources:
|
|
if resources[state]:
|
|
print("{0}: {1}".format(state, self._server_names(
|
|
resources[state].keys())))
|
|
user_input = six.moves.input(
|
|
"Breakpoint reached, continue? Regexp or "
|
|
"Enter=proceed (will clear %s), "
|
|
"C-c=quit interactive mode: "
|
|
% resources['on_breakpoint'].keys()[-1])
|
|
refs = self._input_to_refs(
|
|
user_input.strip(),
|
|
resources['on_breakpoint'].keys())
|
|
self.clear_breakpoints(refs)
|
|
time.sleep(5)
|
|
print('update finished with status {0}'.format(status))
|
|
|
|
def _resources_by_state(self):
|
|
resources = {
|
|
'not_started': {},
|
|
'in_progress': {},
|
|
'on_breakpoint': {},
|
|
'completed': {},
|
|
'failed': {},
|
|
}
|
|
all_resources = self.heatclient.resources.list(
|
|
self.stack.id, nested_depth=self.nested_depth)
|
|
if self.hook_type == 'pre-create':
|
|
hook_reason = 'CREATE paused until Hook pre-create is cleared'
|
|
hook_clear_reason = 'Hook pre-create is cleared'
|
|
else:
|
|
hook_reason = 'UPDATE paused until Hook pre-update is cleared'
|
|
hook_clear_reason = 'Hook pre-update is cleared'
|
|
|
|
stack_change_time = self._stack_change_time()
|
|
|
|
for res in all_resources:
|
|
if self.hook_resource:
|
|
if not fnmatch.fnmatchcase(res.resource_name,
|
|
self.hook_resource):
|
|
continue
|
|
stack_name, stack_id = next(
|
|
x['href'] for x in res.links if
|
|
x['rel'] == 'stack').rsplit('/', 2)[1:]
|
|
try:
|
|
events = self.heatclient.events.list(
|
|
stack_id=stack_id,
|
|
resource_name=res.logical_resource_id,
|
|
sort_dir='asc')
|
|
except heatclient.exc.HTTPNotFound:
|
|
events = []
|
|
state = 'not_started'
|
|
for ev in events:
|
|
# ignore events older than start of the last stack change
|
|
if ev.event_time < stack_change_time:
|
|
continue
|
|
if ev.resource_status_reason == hook_reason:
|
|
state = 'on_breakpoint'
|
|
elif ev.resource_status_reason == hook_clear_reason:
|
|
state = 'in_progress'
|
|
elif ev.resource_status in ('CREATE_IN_PROGRESS',
|
|
'UPDATE_IN_PROGRESS'):
|
|
state = 'in_progress'
|
|
elif ev.resource_status in ('CREATE_COMPLETE',
|
|
'UPDATE_COMPLETE'):
|
|
state = 'completed'
|
|
resources[state][res.physical_resource_id] = res
|
|
|
|
return resources
|
|
|
|
def _stack_change_time(self):
|
|
if self.hook_type == 'pre-create':
|
|
status_reason = 'Stack CREATE started'
|
|
else:
|
|
status_reason = 'Stack UPDATE started'
|
|
events = self.heatclient.events.list(
|
|
stack_id=self.stack.id,
|
|
sort_dir='desc')
|
|
try:
|
|
ev = next(e for e in events if
|
|
e.resource_status_reason == status_reason)
|
|
return ev.event_time
|
|
except StopIteration:
|
|
return None
|
|
|
|
def _server_names(self, deployment_ids):
|
|
return [self._server_name(i) for i in deployment_ids]
|
|
|
|
def _server_name(self, deployment_id):
|
|
name = self.server_names.get(deployment_id)
|
|
if not name:
|
|
if not self.servers:
|
|
self.servers = self._get_servers()
|
|
depl = self.heatclient.software_deployments.get(deployment_id)
|
|
name = next(server.name for server in self.servers if
|
|
server.id == depl.server_id)
|
|
self.server_names[deployment_id] = name
|
|
return name
|
|
|
|
def _get_servers(self):
|
|
servers = self.novaclient.servers.list()
|
|
|
|
# If no servers were found from Nova, we must be using split-stack,
|
|
# so we will have to interrogate Heat for the names and id's.
|
|
if not servers:
|
|
resources = self.heatclient.resources.list(
|
|
self.stack.id, nested_depth=self.nested_depth,
|
|
filters=dict(type="OS::Heat::DeployedServer"))
|
|
for res in resources:
|
|
server = DeployedServer()
|
|
stack_name, stack_id = next(
|
|
x['href'] for x in res.links if
|
|
x['rel'] == 'stack').rsplit('/', 2)[1:]
|
|
stack = self.heatclient.stacks.get(stack_id)
|
|
server.name = next(o['output_value'] for o in stack.outputs if
|
|
o['output_key'] == 'name')
|
|
server.id = res.physical_resource_id
|
|
servers.append(server)
|
|
|
|
return servers
|
|
|
|
def _input_to_refs(self, regexp, refs):
|
|
if regexp:
|
|
try:
|
|
pattern = "\A{0}\Z".format(regexp)
|
|
return [ref for ref in refs if
|
|
re.match(pattern, self._server_name(ref))]
|
|
except re.error as err:
|
|
LOG.warning("'%s' is invalid regular expression: %s",
|
|
regexp.encode('string-escape'), err)
|
|
return []
|
|
else:
|
|
return [refs.pop()]
|