Merge "Max percentage failure support" into stable/train
This commit is contained in:
commit
2d5ee5f8b7
@ -47,6 +47,7 @@ class TripleoBase(StrategyBase):
|
||||
self._play_context = None
|
||||
self._strat_results = []
|
||||
self.noop_task = None
|
||||
self._fail_cache = {}
|
||||
# these were defined in 2.9
|
||||
self._has_hosts_cache = False
|
||||
self._has_hosts_cache_all = False
|
||||
@ -84,6 +85,50 @@ class TripleoBase(StrategyBase):
|
||||
task.name = name
|
||||
self._callback_sent = True
|
||||
|
||||
def _get_fail_percent(self, host):
|
||||
"""Return maximum percentage failure per role"""
|
||||
if host and host in self._fail_cache:
|
||||
return self._fail_cache[host]
|
||||
|
||||
fail_vars = self._variable_manager.get_vars(play=self._iterator._play,
|
||||
host=host,
|
||||
task=None)
|
||||
percent = fail_vars.get('max_fail_percentage', 0)
|
||||
role = fail_vars.get('tripleo_role_name', 'default')
|
||||
self._fail_cache[host] = (percent, role)
|
||||
return (percent, role)
|
||||
|
||||
def _check_fail_percent(self, host, current_failures):
|
||||
"""Check if max fail pourcentage was reached
|
||||
|
||||
When a failure occurs for a host, check if we reached
|
||||
the max percentage of failure for the group in which
|
||||
the host is part from.
|
||||
"""
|
||||
percent, role = self._get_fail_percent(host)
|
||||
current_failed = current_failures.get(role, 1)
|
||||
|
||||
groups = self._inventory.get_groups_dict()
|
||||
group_count = len(groups.get(role, []))
|
||||
if group_count == 0:
|
||||
return True
|
||||
failed_percent = (current_failed / group_count) * 100
|
||||
if failed_percent > percent:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_current_failures(self):
|
||||
"""Return the number of failures per role"""
|
||||
failures = {}
|
||||
for host, _ in self._iterator.get_failed_hosts().items():
|
||||
host_obj = self._inventory.get_host(host)
|
||||
per, role = self._get_fail_percent(host_obj)
|
||||
if role in failures:
|
||||
failures[role] += 1
|
||||
else:
|
||||
failures[role] = 1
|
||||
return failures
|
||||
|
||||
def process_includes(self, host_results, noop=False):
|
||||
"""Handle includes
|
||||
|
||||
|
@ -123,10 +123,12 @@ class StrategyModule(BASE.TripleoBase):
|
||||
function returns True if there were failures and False if
|
||||
there are no failures.
|
||||
"""
|
||||
fail_lookup = self._get_current_failures()
|
||||
if self._any_errors_fatal:
|
||||
for res in results:
|
||||
if ((res.is_failed() or res._task.action == 'meta')
|
||||
and self._iterator.is_failed(res._host)):
|
||||
and self._iterator.is_failed(res._host)
|
||||
and self._check_fail_percent(res._host, fail_lookup)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
@ -334,15 +334,20 @@ class StrategyModule(BASE.TripleoBase):
|
||||
|
||||
failed_hosts = []
|
||||
unreachable_hosts = []
|
||||
fail_lookup = self._get_current_failures()
|
||||
for res in self._strat_results:
|
||||
if ((res.is_failed() or res._task.action == 'meta')
|
||||
and self._iterator.is_failed(res._host)):
|
||||
failed_hosts.append(res._host.name)
|
||||
failed_hosts.append(res._host)
|
||||
elif res.is_unreachable():
|
||||
unreachable_hosts.append(res._host.name)
|
||||
unreachable_hosts.append(res._host)
|
||||
|
||||
# TODO(mwhahaha): handle max_fail_percentage by tripleo role
|
||||
if (self._any_errors_fatal
|
||||
errored = False
|
||||
for host in set(failed_hosts + unreachable_hosts):
|
||||
errored = self._check_fail_percent(host, fail_lookup)
|
||||
if errored:
|
||||
break
|
||||
if (errored and self._any_errors_fatal
|
||||
and (len(failed_hosts) > 0
|
||||
or len(unreachable_hosts) > 0)):
|
||||
result = self._process_failures()
|
||||
|
Loading…
Reference in New Issue
Block a user