Merge "Run Validations with ThreadPoolExecutor" into stable/stein
This commit is contained in:
commit
4d0c4ab51c
|
@ -1,6 +1,7 @@
|
||||||
# The order of packages is significant, because pip processes them in the order
|
# The order of packages is significant, because pip processes them in the order
|
||||||
# of appearance. Changing the order has an impact on the overall integration
|
# of appearance. Changing the order has an impact on the overall integration
|
||||||
# process, which may cause wedges in the gate later.
|
# process, which may cause wedges in the gate later.
|
||||||
sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
|
sphinx>=1.6.2,!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7' # BSD
|
||||||
|
sphinx>=1.6.2,!=1.6.6,!=1.6.7;python_version>='3.4' # BSD
|
||||||
openstackdocstheme>=1.18.1 # Apache-2.0
|
openstackdocstheme>=1.18.1 # Apache-2.0
|
||||||
reno>=2.5.0 # Apache-2.0
|
reno>=2.5.0 # Apache-2.0
|
||||||
|
|
|
@ -25,6 +25,7 @@ extras==1.0.0
|
||||||
fasteners==0.7.0
|
fasteners==0.7.0
|
||||||
fixtures==3.0.0
|
fixtures==3.0.0
|
||||||
flake8==2.5.5
|
flake8==2.5.5
|
||||||
|
futures==3.0.0
|
||||||
futurist==1.2.0
|
futurist==1.2.0
|
||||||
gitdb==0.6.4
|
gitdb==0.6.4
|
||||||
GitPython==1.0.1
|
GitPython==1.0.1
|
||||||
|
|
|
@ -18,3 +18,4 @@ osc-lib>=1.8.0 # Apache-2.0
|
||||||
websocket-client>=0.44.0 # LGPLv2+
|
websocket-client>=0.44.0 # LGPLv2+
|
||||||
tripleo-common>=10.6.1 # Apache-2.0
|
tripleo-common>=10.6.1 # Apache-2.0
|
||||||
cryptography>=2.1 # BSD/Apache-2.0
|
cryptography>=2.1 # BSD/Apache-2.0
|
||||||
|
futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD
|
||||||
|
|
|
@ -18,8 +18,10 @@ import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import pwd
|
import pwd
|
||||||
|
import six
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from osc_lib.command import command
|
from osc_lib.command import command
|
||||||
from osc_lib.i18n import _
|
from osc_lib.i18n import _
|
||||||
|
|
||||||
|
@ -212,6 +214,16 @@ class TripleOValidatorRun(command.Command):
|
||||||
help=_("Execute the validations using Mistral")
|
help=_("Execute the validations using Mistral")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'--workers', '-w',
|
||||||
|
metavar='N',
|
||||||
|
dest='workers',
|
||||||
|
default=1,
|
||||||
|
type=int,
|
||||||
|
help=_("The maximum number of threads that can "
|
||||||
|
"be used to execute the given validations")
|
||||||
|
)
|
||||||
|
|
||||||
extra_vars_group = parser.add_mutually_exclusive_group(required=False)
|
extra_vars_group = parser.add_mutually_exclusive_group(required=False)
|
||||||
|
|
||||||
extra_vars_group.add_argument(
|
extra_vars_group.add_argument(
|
||||||
|
@ -302,6 +314,23 @@ class TripleOValidatorRun(command.Command):
|
||||||
out.get('validation_name'),
|
out.get('validation_name'),
|
||||||
oooutils.indent(out.get('stdout'))))
|
oooutils.indent(out.get('stdout'))))
|
||||||
|
|
||||||
|
def _run_ansible(self, logger, plan, workdir, log_path_dir, playbook,
|
||||||
|
inventory, retries, output_callback, extra_vars,
|
||||||
|
python_interpreter, gathering_policy):
|
||||||
|
rc, output = oooutils.run_ansible_playbook(
|
||||||
|
logger=logger,
|
||||||
|
plan=plan,
|
||||||
|
workdir=workdir,
|
||||||
|
log_path_dir=log_path_dir,
|
||||||
|
playbook=playbook,
|
||||||
|
inventory=inventory,
|
||||||
|
retries=retries,
|
||||||
|
output_callback=output_callback,
|
||||||
|
extra_vars=extra_vars,
|
||||||
|
python_interpreter=python_interpreter,
|
||||||
|
gathering_policy=gathering_policy)
|
||||||
|
return rc, output
|
||||||
|
|
||||||
def _run_validator_run(self, parsed_args):
|
def _run_validator_run(self, parsed_args):
|
||||||
clients = self.app.client_manager
|
clients = self.app.client_manager
|
||||||
LOG = logging.getLogger(__name__ + ".ValidationsRunAnsible")
|
LOG = logging.getLogger(__name__ + ".ValidationsRunAnsible")
|
||||||
|
@ -343,10 +372,11 @@ class TripleOValidatorRun(command.Command):
|
||||||
|
|
||||||
failed_val = False
|
failed_val = False
|
||||||
|
|
||||||
for playbook in playbooks:
|
with ThreadPoolExecutor(max_workers=parsed_args.workers) as executor:
|
||||||
try:
|
LOG.debug(_('Running the validations with Ansible'))
|
||||||
LOG.debug(_('Running the validations with Ansible'))
|
tasks_exec = {
|
||||||
rc, output = oooutils.run_ansible_playbook(
|
executor.submit(
|
||||||
|
self._run_ansible,
|
||||||
logger=LOG,
|
logger=LOG,
|
||||||
plan=parsed_args.plan,
|
plan=parsed_args.plan,
|
||||||
workdir=constants.ANSIBLE_VALIDATION_DIR,
|
workdir=constants.ANSIBLE_VALIDATION_DIR,
|
||||||
|
@ -357,13 +387,18 @@ class TripleOValidatorRun(command.Command):
|
||||||
output_callback='validation_output',
|
output_callback='validation_output',
|
||||||
extra_vars=extra_vars_input,
|
extra_vars=extra_vars_input,
|
||||||
python_interpreter=python_interpreter,
|
python_interpreter=python_interpreter,
|
||||||
gathering_policy='explicit')
|
gathering_policy='explicit'): playbook
|
||||||
print('[SUCCESS] - {}\n{}'.format(playbook,
|
for playbook in playbooks
|
||||||
oooutils.indent(output)))
|
}
|
||||||
|
|
||||||
|
for tk, pl in six.iteritems(tasks_exec):
|
||||||
|
try:
|
||||||
|
rc, output = tk.result()
|
||||||
|
print('[SUCCESS] - {}\n{}'.format(pl, oooutils.indent(output)))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
failed_val = True
|
failed_val = True
|
||||||
LOG.error('[FAILED] - {}\n{}'.format(
|
LOG.error('[FAILED] - {}\n{}'.format(
|
||||||
playbook, oooutils.indent(e.args[0])))
|
pl, oooutils.indent(e.args[0])))
|
||||||
|
|
||||||
LOG.debug(_('Removing static tripleo ansible inventory file'))
|
LOG.debug(_('Removing static tripleo ansible inventory file'))
|
||||||
oooutils.cleanup_tripleo_ansible_inventory_file(
|
oooutils.cleanup_tripleo_ansible_inventory_file(
|
||||||
|
|
Loading…
Reference in New Issue