Refactor Ansible runner

performa/engine/ansible_runner.py

@@ -14,12 +14,13 @@
 # limitations under the License.
 
 from collections import namedtuple
+import copy
 
 from ansible.executor import task_queue_manager
 from ansible import inventory
 from ansible.parsing import dataloader
 from ansible.playbook import play
-from ansible.plugins import callback
+from ansible.plugins import callback as callback_pkg
 from ansible.vars import VariableManager
 from oslo_log import log as logging
 
@@ -28,7 +29,22 @@ from performa.engine import utils
 LOG = logging.getLogger(__name__)
 
 
-class MyCallback(callback.CallbackBase):
+def _light_rec(result):
+    for r in result:
+        c = copy.deepcopy(r)
+        if 'records' in c:
+            del c['records']
+        if 'series' in c:
+            del c['series']
+        yield c
+
+
+def _log_result(result):
+    # todo check current log level before doing heavy things
+    LOG.debug('Execution result (filtered): %s', list(_light_rec(result)))
+
+
+class MyCallback(callback_pkg.CallbackBase):
 
     CALLBACK_VERSION = 2.0
     CALLBACK_TYPE = 'stdout'
@@ -70,79 +86,65 @@ Options = namedtuple('Options',
                       'become_method', 'become_user', 'verbosity', 'check'])
 
 
-def _run(play_source, host_list):
+class AnsibleRunner(object):
+    def __init__(self, remote_user='developer', forks=100):
+        super(AnsibleRunner, self).__init__()
 
-    LOG.debug('Running play: %s on hosts: %s', play_source, host_list)
+        module_path = utils.resolve_relative_path('performa/modules')
+        self.options = Options(
+            connection='smart', password='swordfish', module_path=module_path,
+            forks=forks, remote_user=remote_user, private_key_file=None,
+            ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None,
+            scp_extra_args=None, become=None, become_method='sudo',
+            become_user='root', verbosity=100, check=False)
 
-    variable_manager = VariableManager()
+    def _run_play(self, play_source):
-    loader = dataloader.DataLoader()
+        LOG.debug('Running play: %s', play_source)
-    module_path = utils.resolve_relative_path('performa/modules')
 
-    options = Options(connection='smart', password='swordfish',
+        host_list = play_source['hosts']
-                      module_path=module_path,
-                      forks=100, remote_user='developer',
-                      private_key_file=None,
-                      ssh_common_args=None, ssh_extra_args=None,
-                      sftp_extra_args=None, scp_extra_args=None, become=None,
-                      become_method=None, become_user=None, verbosity=100,
-                      check=False)
-    passwords = dict(vault_pass='secret')
 
-    # create inventory and pass to var manager
+        loader = dataloader.DataLoader()
-    inventory_inst = inventory.Inventory(loader=loader,
+        variable_manager = VariableManager()
-                                         variable_manager=variable_manager,
+        inventory_inst = inventory.Inventory(loader=loader,
-                                         host_list=host_list)
+                                             variable_manager=variable_manager,
-    variable_manager.set_inventory(inventory_inst)
+                                             host_list=host_list)
+        variable_manager.set_inventory(inventory_inst)
+        passwords = dict(vault_pass='secret')
 
-    # create play
+        # create play
-    play_inst = play.Play().load(play_source,
+        play_inst = play.Play().load(play_source,
-                                 variable_manager=variable_manager,
+                                     variable_manager=variable_manager,
-                                 loader=loader)
+                                     loader=loader)
 
-    storage = []
+        storage = []
-    callback = MyCallback(storage)
+        callback = MyCallback(storage)
 
-    # actually run it
+        # actually run it
-    tqm = None
+        tqm = None
-    try:
+        try:
-        tqm = task_queue_manager.TaskQueueManager(
+            tqm = task_queue_manager.TaskQueueManager(
-            inventory=inventory_inst,
+                inventory=inventory_inst,
-            variable_manager=variable_manager,
+                variable_manager=variable_manager,
-            loader=loader,
+                loader=loader,
-            options=options,
+                options=self.options,
-            passwords=passwords,
+                passwords=passwords,
-            stdout_callback=callback,
+                stdout_callback=callback,
-        )
+            )
-        tqm.run(play_inst)
+            tqm.run(play_inst)
-    finally:
+        finally:
-        if tqm is not None:
+            if tqm is not None:
-            tqm.cleanup()
+                tqm.cleanup()
 
-    LOG.debug('Execution result: %s', storage)
+        _log_result(storage)
-    return storage
 
+        return storage
 
-def run_command(command, host_list):
+    def run(self, playbook):
-    hosts = ','.join(host_list) + ','
+        result = []
-    # tasks = [dict(action=dict(module='shell', args=command))]
-    tasks = [{'command': command}]
 
-    play_source = dict(
+        for play_source in playbook:
-        hosts=host_list,
+            play_source['gather_facts'] = 'no'
-        gather_facts='no',
-        tasks=tasks,
-    )
 
-    return _run(play_source, hosts)
+            result += self._run_play(play_source)
 
-
+        return result
-def run_playbook(playbook):
-    result = []
-
-    for play_source in playbook:
-        hosts = play_source['hosts']
-        play_source['gather_facts'] = 'no'
-
-        result += (_run(play_source, hosts))
-
-    return result

performa/engine/config.py

@@ -67,6 +67,11 @@ MAIN_OPTS = [
                default=utils.env('PERFORMA_MONGO_DB'),
                required=True,
                help='Mongo DB, defaults to env[PERFORMA_MONGO_DB].'),
+    cfg.StrOpt('remote-user',
+               default=utils.env('PERFORMA_REMOTE_USER'),
+               required=True,
+               help='User for connecting to remote systems, '
+                    'defaults to env[PERFORMA_REMOTE_USER].'),
     cfg.Opt('hosts',
             type=Yaml(),
             default=utils.env('PERFORMA_HOSTS'),

performa/engine/main.py

@@ -19,6 +19,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import yaml
 
+from performa.engine import ansible_runner
 from performa.engine import config
 from performa.engine import player
 from performa.engine import report
@@ -53,7 +54,9 @@ def main():
         tag = utils.random_string()
         LOG.info('Using auto-generated tag "%s"', tag)
 
-    records, series = player.play_scenario(scenario, tag)
+    runner = ansible_runner.AnsibleRunner(remote_user=cfg.CONF.remote_user)
+
+    records, series = player.play_scenario(runner, scenario, tag)
 
     storage.store_data(cfg.CONF.mongo_url, cfg.CONF.mongo_db, records, series)
 

performa/engine/player.py

@@ -15,19 +15,13 @@
 
 import copy
 
-from oslo_config import cfg
 from oslo_log import log as logging
 
-from performa.engine import ansible_runner
 from performa.engine import utils
 
 LOG = logging.getLogger(__name__)
 
 
-def run_command(command):
-    return ansible_runner.run_command(command, cfg.CONF.hosts)
-
-
 def _pick_tasks(tasks, matrix):
     matrix = matrix or {}
 
@@ -42,11 +36,11 @@ def _pick_tasks(tasks, matrix):
             yield parametrized_task
 
 
-def play_setup(setup):
+def play_setup(runner, setup_playbook):
-    ansible_runner.run_playbook(setup)
+    runner.run(setup_playbook)
 
 
-def play_execution(execution_playbook):
+def play_execution(runner, execution_playbook):
     records = []
     series = []
 
@@ -59,7 +53,7 @@ def play_execution(execution_playbook):
                 'hosts': play['hosts'],
                 'tasks': [task],
             }
-            command_results = ansible_runner.run_playbook([task_play])
+            command_results = runner.run([task_play])
 
             for command_result in command_results:
                 if command_result.get('status') == 'OK':
@@ -91,17 +85,17 @@ def add_tag(records, tag):
         r['tag'] = tag
 
 
-def play_scenario(scenario, tag):
+def play_scenario(runner, scenario, tag):
     records = []
     series = []
 
     if 'setup' in scenario:
-        play_setup(scenario['setup'])
+        play_setup(runner, scenario['setup'])
 
     if 'execution' in scenario:
         execution = scenario['execution']
 
-        records, series = play_execution(execution)
+        records, series = play_execution(runner, execution)
         add_tag(records, tag)
         add_tag(series, tag)
 

performa/scenarios/db/sysbench.yaml

@@ -10,14 +10,10 @@ setup:
     - name: installing sysbench
       apt: name=sysbench
       become: yes
-      become_user: root
-      become_method: sudo
     - name: installing atop
       apt:
         name: atop, daemon
       become: yes
-      become_user: root
-      become_method: sudo
   -
     hosts: $target
     tasks: